Strip domain from url

repository/repository.go

@@ -2,6 +2,7 @@ package repository
 
 import (
 	"fmt"
+	"net/url"
 	"news-master/app"
 	"news-master/datamodels/dto"
 	"news-master/datamodels/entity"
@@ -82,11 +83,19 @@ func CreateResult(result dto.Article) {
 		Language:    result.Language,
 		Country:     pq.StringArray(result.Country),
 		Category:    pq.StringArray(result.Category),
-		Site:        result.SourceUrl,
+		Site:        getDomain(result.SourceUrl),
 	}
 	db().Create(&article)
 }
 
+func getDomain(rawURL string) string {
+	parsedURL, err := url.Parse(rawURL)
+	if err != nil {
+		return fmt.Sprintf("INVALID_RAW_URL_%v", rawURL)
+	}
+	return parsedURL.Host
+}
+
 func GetActiveSites() []entity.Site {
 	var sites []entity.Site
 	db().Where(entity.Site{Active: true}).Find(&sites)

tasks/tasks.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"news-master/app"
 	"news-master/auth"
 	notification "news-master/cmd/process"
@@ -40,6 +41,10 @@ func FetchNewsTask() {
 			return
 		}
 
+		for i := range response.Results {
+			response.Results[i].SourceUrl = getDomain(response.Results[i].SourceUrl)
+		}
+
 		unmarshalErr := json.Unmarshal(body, &response)
 
 		if unmarshalErr != nil {
@@ -56,6 +61,14 @@ func FetchNewsTask() {
 	}
 }
 
+func getDomain(rawURL string) string {
+	parsedURL, err := url.Parse(rawURL)
+	if err != nil {
+		return fmt.Sprintf("INVALID_RAW_URL_%v", rawURL)
+	}
+	return parsedURL.Host
+}
+
 func SendNewsletter() {
 	subscriptions := repository.GetSubscriptionsToProcess()