From d8c857959bf7c2fbd24cca135027fe0f0ceaa925 Mon Sep 17 00:00:00 2001 From: Angad Behl <77907286+slashtechno@users.noreply.github.com> Date: Wed, 19 Jun 2024 18:43:30 +0000 Subject: [PATCH] feat!: Publish from Markdown --- .vscode/launch.json | 24 +++++++++- cobra/.gitignore | 3 +- cobra/cmd/platforms.go | 72 ++++++++++++++++++++++++---- cobra/cmd/publish.go | 10 +++- cobra/output_markdown/hello-world.md | 6 --- go.mod | 3 ++ go.sum | 7 ++- 7 files changed, 105 insertions(+), 20 deletions(-) delete mode 100644 cobra/output_markdown/hello-world.md diff --git a/.vscode/launch.json b/.vscode/launch.json index b9c9d03..1b85eea 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -16,9 +16,17 @@ "request": "launch", "program": "${workspaceFolder}/cobra", // https://www.blogger.com/edit-profile.g - "args": ["publish", "blogger", "${input:PostURL}", "blogger"], + "args": ["publish", "blogger", "${input:PostURL}", "${input:Destination}"], // "envFile": "${workspaceFolder}/cobra/.env", "console": "integratedTerminal" + }, + { + "name": "Publish from Markdown", + "type": "go", + "request": "launch", + "program": "${workspaceFolder}/cobra", + "args": ["publish", "markdown", "${input:MarkdownPath}", "${input:Destination}"], + "console": "integratedTerminal" } ], "inputs": [ @@ -26,7 +34,19 @@ "id": "PostURL", "type": "promptString", "description": "Enter the URL of the Blogger post to publish", - "default": "https://itsfrommars.blogspot.com/2024/06/hello-world_78.html" + "default": "https://itsfrommars.blogspot.com/2024/06/hello-world_11.html" + }, + { + "id": "MarkdownPath", + "type": "promptString", + "description": "Enter the path of the Markdown file to publish", + "default": "hello-world.md" + }, + { + "id": "Destination", + "type": "promptString", + "description": "Enter the destination to publish", + "default": "markdown" } ] } \ No newline at end of file diff --git a/cobra/.gitignore b/cobra/.gitignore index ab8b69c..6b97628 100644 --- a/cobra/.gitignore +++ b/cobra/.gitignore @@ -1 +1,2 @@ -config.toml \ No newline at end of file +config.toml +*_markdown/ \ No newline at end of file diff --git a/cobra/cmd/platforms.go b/cobra/cmd/platforms.go index 491d4c5..7230b8d 100644 --- a/cobra/cmd/platforms.go +++ b/cobra/cmd/platforms.go @@ -1,12 +1,15 @@ package cmd import ( + "bytes" "fmt" "os" "path/filepath" "strings" "github.com/gosimple/slug" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/parser" "gopkg.in/yaml.v2" md "github.com/JohannesKaufmann/html-to-markdown" @@ -14,6 +17,7 @@ import ( "github.com/go-resty/resty/v2" "github.com/slashtechno/cross-blogger/cobra/pkg/oauth" "github.com/spf13/afero" + "go.abhg.dev/goldmark/frontmatter" ) type Destination interface { @@ -32,6 +36,12 @@ type PushPullOptions struct { AccessToken string BlogId string PostUrl string + Filepath string +} + +type Frontmatter struct { + Title string `yaml:"title"` + CanonicalUrl string `yaml:"canonicalURL"` } type PostData struct { @@ -251,15 +261,12 @@ func (m Markdown) Push(data PostData, options PushPullOptions) error { // After the function returns, close the file defer file.Close() // Create the frontmatter - frontmatter := struct { - Title string `yaml:"title"` - CanonicalUrl string `yaml:"canonicalUrl"` - }{ + postFrontmatter := Frontmatter{ Title: data.Title, CanonicalUrl: data.CanonicalUrl, } // Convert the frontmatter to YAML - frontmatterYaml, err := yaml.Marshal(frontmatter) + frontmatterYaml, err := yaml.Marshal(postFrontmatter) if err != nil { return err } @@ -273,8 +280,57 @@ func (m Markdown) Push(data PostData, options PushPullOptions) error { } func (m Markdown) Pull(options PushPullOptions) (PostData, error) { - log.Info("Markdown pull called", "options", options) - return PostData{}, nil + // Get the file path + fs := afero.NewOsFs() + // Treat the post path as relative to the content dir + // However, if the content dir does not exist or the file is not found, treat the post path as a normal path without the content dir + filePath := filepath.Join(m.ContentDir, options.Filepath) + if _, err := fs.Stat(filePath); os.IsNotExist(err) { + filePath = options.Filepath + } + // Read the file + data, err := afero.ReadFile(fs, filePath) + if err != nil { + return PostData{}, err + } + markdown := string(data) + // Convert the markdown to HTML with Goldmark + // Use the Frontmatter extension to get the frontmatter + mdParser := goldmark.New(goldmark.WithExtensions(&frontmatter.Extender{})) + ctx := parser.NewContext() + var buf bytes.Buffer + err = mdParser.Convert([]byte(markdown), &buf, parser.WithContext(ctx)) + if err != nil { + return PostData{}, err + } + // Get the frontmatter + markdownFrontmatter := Frontmatter{} + frontmatterData := frontmatter.Get(ctx) + if err := frontmatterData.Decode(&markdownFrontmatter); err != nil { + return PostData{}, err + } + // Check if title and canonical URL are set + if markdownFrontmatter.Title == "" { + return PostData{}, fmt.Errorf("title is required in frontmatter") + } + if markdownFrontmatter.CanonicalUrl == "" { + log.Warn("canonical_url is not set in frontmatter") + } + // Convert the HTML to Markdown + html := buf.String() + // The frontmatter is stripped before converting to HTML + // Just convert the HTML to Markdown so the Markdown doesn't have the frontmatter (otherwise it would be duplicated) + markdown, err = md.NewConverter("", true, nil).ConvertString(html) + if err != nil { + return PostData{}, err + } + return PostData{ + Title: markdownFrontmatter.Title, + Html: html, + Markdown: markdown, + CanonicalUrl: markdownFrontmatter.CanonicalUrl, + }, nil + } func CreateDestination(destMap map[string]interface{}) (Destination, error) { @@ -333,7 +389,7 @@ func CreateSource(sourceMap map[string]interface{}) (Source, error) { Name: name, BlogUrl: blogUrl, }, nil - case "file": + case "markdown": // If the content_dir is not set, set it to null as its not required contentDir, _ := sourceMap["content_dir"].(string) return Markdown{ diff --git a/cobra/cmd/publish.go b/cobra/cmd/publish.go index 8107629..ec8f38b 100644 --- a/cobra/cmd/publish.go +++ b/cobra/cmd/publish.go @@ -98,7 +98,9 @@ var publishCmd = &cobra.Command{ PostUrl: args[1], } case "markdown": - log.Fatal("Markdown source not implemented") + options = PushPullOptions{ + Filepath: args[1], + } } // Pull the data from the source postData, err := source.Pull(options) @@ -127,6 +129,11 @@ var publishCmd = &cobra.Command{ found = false } if found { + // Check if this is a dry run + if viper.GetBool("dry-run") { + log.Info("Dry run - not pushing data") + continue + } err := destination.Push(postData, options) if err != nil { log.Fatal(err) @@ -140,7 +147,6 @@ var publishCmd = &cobra.Command{ func init() { RootCmd.AddCommand(publishCmd) - // Perhaps add a -f flag to force overwrite posts/files if they already exist publishCmd.Flags().StringP("title", "t", "", "Specify custom title instead of using the default") publishCmd.Flags().BoolP("dry-run", "r", false, "Don't actually publish") publishCmd.Flags().String("google-client-id", "", "Google OAuth client ID") diff --git a/cobra/output_markdown/hello-world.md b/cobra/output_markdown/hello-world.md deleted file mode 100644 index a536674..0000000 --- a/cobra/output_markdown/hello-world.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: Hello, world! -canonicalUrl: http://itsfrommars.blogspot.com/2022/08/hello-world.html ---- - -I created this post from the Google API. Hello, world! \ No newline at end of file diff --git a/go.mod b/go.mod index abffaec..60474f6 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,8 @@ require ( github.com/spf13/viper v1.19.0 github.com/subosito/gotenv v1.6.0 github.com/tidwall/gjson v1.16.0 + github.com/yuin/goldmark v1.7.2 + go.abhg.dev/goldmark/frontmatter v0.2.0 golang.org/x/oauth2 v0.18.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -24,6 +26,7 @@ require ( require ( cloud.google.com/go/compute v1.24.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/BurntSushi/toml v1.2.1 // indirect github.com/PuerkitoBio/goquery v1.8.1 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect diff --git a/go.sum b/go.sum index 1936e75..2797f0d 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1Yl cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/JohannesKaufmann/html-to-markdown v1.4.0 h1:uaIPDub6VrBsQP0r5xKjpPo9lxMcuQF1L1pT6BiBdmw= github.com/JohannesKaufmann/html-to-markdown v1.4.0/go.mod h1:3p+lDUqSw+cxolZl7OINYzJ70JHXogXjyCl9UnMQ5gU= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= @@ -127,8 +129,11 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.2 h1:NjGd7lO7zrUn/A7eKwn5PEOt4ONYGqpxSEeZuduvgxc= +github.com/yuin/goldmark v1.7.2/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= +go.abhg.dev/goldmark/frontmatter v0.2.0 h1:P8kPG0YkL12+aYk2yU3xHv4tcXzeVnN+gU0tJ5JnxRw= +go.abhg.dev/goldmark/frontmatter v0.2.0/go.mod h1:XqrEkZuM57djk7zrlRUB02x8I5J0px76YjkOzhB4YlU= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=