-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* initial commito * change domain to libgen * wip * wip * wip * rename packages * add current doc and mirror * add downloader package, some things still brokne * fix downloading * clean up * fix parsing of title * delete libgen * remove replace
- Loading branch information
1 parent
d8d0980
commit 9ff5698
Showing
13 changed files
with
665 additions
and
345 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
package api | ||
package book | ||
|
||
type Book struct { | ||
ID string | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
package console | ||
|
||
import ( | ||
"fmt" | ||
|
||
"github.com/fatih/color" | ||
) | ||
|
||
// TODO: Refactor this to just print directly instead | ||
// of doing fmt.Println(console.Hihlight()) and so on... | ||
func Higlight(format string, a ...any) string { | ||
magenta := color.New(color.FgHiWhite).Add(color.BgBlack).SprintFunc() | ||
return magenta(fmt.Sprintf(format, a...)) | ||
} | ||
|
||
func Error(format string, a ...any) string { | ||
red := color.New(color.FgRed).SprintFunc() | ||
return red(fmt.Sprintf(format, a...)) | ||
} | ||
|
||
func Info(format string, a ...any) string { | ||
yellow := color.New(color.FgYellow).SprintFunc() | ||
return yellow(fmt.Sprintf(format, a...)) | ||
} | ||
|
||
func Success(format string, a ...any) string { | ||
green := color.New(color.FgHiGreen).SprintFunc() | ||
return green(fmt.Sprintf(format, a...)) | ||
} | ||
|
||
func Normal(format string, a ...any) string { | ||
white := color.New(color.FgWhite).SprintFunc() | ||
return white(fmt.Sprintf(format, a...)) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,130 @@ | ||
package documentparser | ||
|
||
import ( | ||
"fmt" | ||
"io" | ||
"net/http" | ||
"strings" | ||
|
||
"github.com/PuerkitoBio/goquery" | ||
"github.com/laureanray/clibgen/internal/book" | ||
) | ||
|
||
type CurrentDocumentParser struct { | ||
doc *goquery.Document | ||
} | ||
|
||
func NewCurrentDocumentParser(document *goquery.Document) *CurrentDocumentParser { | ||
return &CurrentDocumentParser{doc: document} | ||
} | ||
|
||
func NewCurrentDocumentParserFromReader(r io.Reader) *CurrentDocumentParser { | ||
document, _ := goquery.NewDocumentFromReader(r) | ||
return &CurrentDocumentParser{doc: document} | ||
} | ||
|
||
func findTitle(selection *goquery.Selection) string { | ||
var title string | ||
selection.Find("a").EachWithBreak(func(v int, s *goquery.Selection) bool { | ||
if s.Text() != "" && len(s.Text()) > 1 { | ||
title = s.Text() | ||
// Break out of the loop | ||
return false | ||
} | ||
return true | ||
}) | ||
|
||
return title | ||
} | ||
|
||
func (cdp *CurrentDocumentParser) GetBookDataFromDocument() []book.Book { | ||
var books []book.Book | ||
cdp.doc.Find("#tablelibgen > tbody > tr").Each(func(resultsRow int, bookRow *goquery.Selection) { | ||
var id, author, title, publisher, extension, year, fileSize string | ||
var mirrors []string | ||
if resultsRow != 0 { | ||
bookRow.Find("td").Each(func(column int, columnSelection *goquery.Selection) { | ||
switch column { | ||
case 0: | ||
title = findTitle(columnSelection) | ||
case 1: | ||
author = columnSelection.Text() | ||
case 2: | ||
publisher = columnSelection.Text() | ||
case 3: | ||
year = columnSelection.Text() | ||
case 6: | ||
fileSize = columnSelection.Text() | ||
case 7: | ||
extension = columnSelection.Text() | ||
case 8: | ||
columnSelection.Find("a").Each(func(linkCol int, link *goquery.Selection) { | ||
href, hrefExists := link.Attr("href") | ||
if hrefExists { | ||
mirrors = append(mirrors, href) | ||
} | ||
}) | ||
} | ||
}) | ||
books = append(books, book.Book{ | ||
ID: id, | ||
Author: author, | ||
Year: year, | ||
Title: title, | ||
Publisher: publisher, | ||
Extension: extension, | ||
Mirrors: mirrors, | ||
FileSize: fileSize, | ||
}) | ||
} | ||
}) | ||
return books | ||
} | ||
|
||
func (cdp *CurrentDocumentParser) getDownloadLinkFromDocument() (string, bool) { | ||
return cdp.doc.Find("#main a").First().Attr("href") | ||
} | ||
|
||
func (cdp *CurrentDocumentParser) getBookTitleFromSelection(selection *goquery.Selection) string { | ||
var title string | ||
selection.Find("a").Each(func(v int, s *goquery.Selection) { | ||
_, exists := s.Attr("title") | ||
if exists { | ||
title = s.Text() | ||
} | ||
}) | ||
selection.Find("a > font").Each(func(v int, s *goquery.Selection) { | ||
a := s.Text() | ||
title = strings.ReplaceAll(title, a, "") | ||
}) | ||
return title | ||
} | ||
|
||
func (cdp *CurrentDocumentParser) GetDirectDownloadLink(selectedBook book.Book) string { | ||
fmt.Println("Obtaining direct download link") | ||
|
||
// TODO Implement retry? | ||
link := selectedBook.Mirrors[0] | ||
|
||
resp, err := http.Get(link) | ||
defer func(Body io.ReadCloser) { | ||
err := Body.Close() | ||
|
||
if err != nil { | ||
fmt.Println("Error closing body:", err) | ||
} | ||
}(resp.Body) | ||
|
||
if err != nil { | ||
fmt.Println("Error getting response:", err) | ||
} | ||
|
||
directDownloadLink, exists := | ||
NewCurrentDocumentParserFromReader(resp.Body).getDownloadLinkFromDocument() | ||
|
||
if exists { | ||
return directDownloadLink | ||
} | ||
|
||
return "" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
package documentparser | ||
|
||
import ( | ||
"github.com/laureanray/clibgen/internal/book" | ||
) | ||
|
||
// type Page struct { | ||
// doc *goquery.Document | ||
// } | ||
|
||
type DocumentParser interface { | ||
GetBookDataFromDocument() []book.Book | ||
GetDownloadLinkFromDocument() (string, bool) | ||
} | ||
|
Oops, something went wrong.