Merge pull request #28 from hoshsadiq/feature/epg-improvements

Improve EPG handling + improve canonical naming, country matching and…

README.md

@@ -47,9 +47,16 @@ providers:
           - Name == "USA CNN"
           - Name == "CNN"
           - Name == "CNN HD"
+epg_providers:
+  - url: file:///path/to/epg.xml
+    channel_id_renames:
+      replacement: find # key = what to replace it with, value = what to find
+      bbc.uk: "BBC One"
 ```
 
-##### The meaning of the config options are as follows:
+#### The meaning of the config options are as follows:
+
+##### Core config
 - `core.server_listen` (`string`)
 
     If set, this will run as a server, rather a single run. If you want single runs, you can omit this option. See the arguments to specify the output.
@@ -74,6 +81,8 @@ providers:
 
     The order to put the categories in.
 
+##### IPTV providers
+
 - `providers`
 
     This is a list of providers of where to retrieve M3U lists. This is an array (see example above).
@@ -137,6 +146,20 @@ providers:
 
     A list of filters to limit this providers setter to. The same logic applies as the above filters method, and thus again, must return true/false. If true, it will run the setters.
 
+##### EPG providers
+
+- `epg_providers`
+
+    This is a list of providers of where to retrieve EPG data from. Each entry in here must be able to retrieve _valid_ XMLTV data. If it unable to decode the full XML, it will skip over it. This is an array (see example above).
+
+- `epg_providers.url` (`string`)
+
+    The URL of where to retrieve the EPG data. This can start with `file://` to retrieve a list from a local file system (this must be an absolute path).
+
+- `epg_providers.channel_id_renames` (`map`)
+
+    This is a key value pair of channel IDs to rename within the XMLTV. This is useful in case the EPG data's channel IDs don't match the channel IDs in the M3U files. This will change the ID.
+
 ##### For filters, name and setters, the following functions are available:
 
 - `strlen(text string) int`
@@ -166,17 +189,30 @@ providers:
 
     Will turn the text in `subject` into a title, by capitalising all words, and also ensures all letters in SD/HD/FHD are capitalised.
 
+- `upper_words(subject string, word string...) string`
+
+    Will turn the text `word` in `subject` into uppercase. Argument `word` can be repeated as multiple times.
+
+- `starts_with(subject string, prefix string...) bool`
+
+    Will return true if the text in `subject` starts with the text in `prefix`.
+
+- `endss_with(subject string, suffix string...) bool`
+
+    Will return true if the text in `subject` ends with the text in `suffix`.
+
 ##### Additionally, the following variables are available:
 
-|variable|content|
-|--------|-------|
-|`ChNo`|The channel number|
-|`Id`|The ID to sync up with XMLTV|
-|`Name`|This is the channel name|
-|`Uri`|The URL for the stream|
-|`Duration`|The duration of the stream, this is usually -1 due to Live TV being being.. well.. live.|
-|`Logo`|The logo (can be either a url or a base64 data string)|
-|`Group`|The group category|
+|variable|content|m3u tag mapping|
+|--------|-------|-------|
+|`ChNo`|The channel number|`tvg-chno`|
+|`Id`|The ID to sync up with XMLTV|`tvg-id`|
+|`Name`|This is the channel name|`tvg-name`|
+|`Uri`|The URL for the stream|The URL (not a tag)|
+|`Duration`|The duration of the stream, this is usually -1 due to Live TV being being.. well.. live.|The duration (not a tag)|
+|`Logo`|The logo (can be either a url or a base64 data string)|`tvg-logo`|
+|`Language`|The language of the stream|`tvg-language`|
+|`Group`|The group category|`group-title`|
 
 ##### Generic expression syntax
 
@@ -222,6 +258,6 @@ The following server endpoints are available for use:
   This is used to force the application to retrieve the latest version of all the providers. This is an asynchronous operation, and will return 204 on success.
 
 ## Future plans
-The idea behind this is to a be one stop shop for generating both xmltv and m3u files from any source.
-This will eventually add support for xml, and will automatically try and match up channels and EPG data should this not exist.
+The idea behind this is to be one stop shop for generating both xmltv and m3u files from any source.
+This will eventually add support for xml, and will automatically try to match up channels and EPG data should this not exist.
 Any other ideas you have? Feel free to raise a ticket.

config/config.go

@@ -9,7 +9,8 @@ import (
 var log = logger.Get()
 
 type EpgProvider struct {
-	Uri string
+	Uri              string
+	ChannelIdRenames map[string]string `yaml:"channel_id_renames"` // key = new_id, value = old_id
 }
 
 type Config struct {
@@ -102,16 +103,6 @@ type Setter struct {
 	Filters []string
 }
 
-type Replacement struct {
-	Name       []*Replacer
-	Attributes map[string][]*Replacer
-}
-
-type Replacer struct {
-	Find    string
-	Replace string
-}
-
 var config *Config
 
 func New(filepath string) (*Config, error) {

m3u/constants.go

@@ -2,6 +2,8 @@ package m3u
 
 import "strings"
 
+var stopWords = `VIP|TV|The`
+
 var definitions = `SD|HD|FHD`
 
 var definitionOverrides = map[string]string{

m3u/evaluator.go

@@ -80,6 +80,8 @@ func getEvaluatorFunctions() map[string]goval.ExpressionFunction {
 		"tvg_id":      evaluatorToTvgId,
 		"title":       evaluatorTitle,
 		"upper_words": evaluatorUpperWord,
+		"starts_with": evaluatorStartsWith,
+		"ends_with":   evaluatorEndsWith,
 	}
 }
 
@@ -138,6 +140,20 @@ func evaluatorUpperWord(args ...interface{}) (interface{}, error) {
 	return strings.TrimSpace(subject), nil
 }
 
+func evaluatorEndsWith(args ...interface{}) (interface{}, error) {
+	subject := args[0].(string)
+	suffix := args[1].(string)
+
+	return strings.HasSuffix(subject, suffix), nil
+}
+
+func evaluatorStartsWith(args ...interface{}) (interface{}, error) {
+	subject := args[0].(string)
+	prefix := args[1].(string)
+
+	return strings.HasSuffix(subject, prefix), nil
+}
+
 func regexWordCallback(subject string, word string, callback func(string) string) string {
 	re := cache.Regexp(`(?i)\b(` + word + `)\b`)
 

m3u/m3u_test.go

@@ -183,7 +183,7 @@ func runTest(path string, t *testing.T, testData simpleTest, ext string, conf *c
 
 	if string(expectedStreams) != string(actualStreams) {
 		t.Logf("Test %s failed.", path)
-		t.Logf("  Expected streans: %s", expectedStreams)
+		t.Logf("  Expected streams: %s", expectedStreams)
 		t.Logf("  Got:              %s", actualStreams)
 		t.Fail()
 	}

m3u/process.go

@@ -141,9 +141,12 @@ func getUri(uri string) (*http.Response, error) {
 }
 
 func getEpg(providers []*config.EpgProvider) (*xmltv.XMLTV, error) {
-	var epg xmltv.XMLTV
+	var epgs = make([]xmltv.XMLTV, len(providers))
+	var newEpg xmltv.XMLTV
+	totalChannels := 0
+	totalProgrammes := 0
 
-	for _, provider := range providers {
+	for i, provider := range providers {
 		resp, err := getUri(provider.Uri)
 		if err != nil {
 			return nil, err
@@ -155,16 +158,28 @@ func getEpg(providers []*config.EpgProvider) (*xmltv.XMLTV, error) {
 			}
 		}()
 
-		err = xmltv.Load(resp.Body, &epg)
+		newEpg = xmltv.XMLTV{}
+		err = xmltv.Load(resp.Body, &newEpg)
 		if err != nil {
 			return nil, err
 		}
+		applyEpgIdRenames(&newEpg, provider.ChannelIdRenames)
+		epgs[i] = newEpg
+		totalChannels += len(newEpg.Channels)
+		totalProgrammes += len(newEpg.Programmes)
 	}
 
-	var channels = make(map[string]*xmltv.Channel, len(epg.Channels))
+	allChannels := make([]*xmltv.Channel, 0, totalChannels)
+	allProgrammes := make([]*xmltv.Programme, 0, totalChannels)
+	for _, epg := range epgs {
+		allChannels = append(allChannels, epg.Channels...)
+		allProgrammes = append(allProgrammes, epg.Programmes...)
+	}
+
+	var channels = make(map[string]*xmltv.Channel, len(allChannels))
 	var nameIdMapping = make(map[string]string)
 
-	for _, c := range epg.Channels {
+	for _, c := range allChannels {
 		channel, ok := channels[c.ID]
 		var found = false
 		if !ok {
@@ -204,7 +219,23 @@ func getEpg(providers []*config.EpgProvider) (*xmltv.XMLTV, error) {
 
 	log.Info("Finished loading EPG")
 
-	return &epg, nil
+	return &xmltv.XMLTV{Programmes: allProgrammes, Channels: allChannels}, nil
+}
+
+func applyEpgIdRenames(epg *xmltv.XMLTV, renames map[string]string) {
+	for newId, oldId := range renames {
+		for _, chann := range epg.Channels {
+			if chann.ID == oldId {
+				chann.ID = newId
+			}
+		}
+
+		for _, programme := range epg.Programmes {
+			if programme.Channel == oldId {
+				programme.Channel = newId
+			}
+		}
+	}
 }
 
 func setMeta(mainCountry string, left *Stream, right *Stream) {

m3u/rename.go

@@ -11,9 +11,6 @@ func setSegmentValues(ms *Stream, epgChannel *xmltv.Channel, setters []*config.S
 	var newValue string
 	var err error
 
-	ms.meta.country = findCountry(ms)
-	ms.meta.definition = findDefinition(ms)
-	ms.meta.canonicalName = canonicaliseName(ms.Name)
 	ms.meta.originalName = ms.Name
 	ms.meta.originalId = ms.Id
 	ms.meta.epgChannel = epgChannel
@@ -95,6 +92,10 @@ func setSegmentValues(ms *Stream, epgChannel *xmltv.Channel, setters []*config.S
 			}
 		}
 	}
+
+	ms.meta.country = findCountry(ms)
+	ms.meta.definition = findDefinition(ms)
+	ms.meta.canonicalName = canonicaliseName(ms.Name)
 }
 
 func addDisplayNameToChannel(epgChannel *xmltv.Channel, newValue string) {
@@ -112,13 +113,22 @@ func addDisplayNameToChannel(epgChannel *xmltv.Channel, newValue string) {
 }
 
 func findCountry(stream *Stream) string {
-	if stream.Id != "" && strings.Count(stream.Id, ".") == 1 {
-		return strings.ToUpper(strings.Split(stream.Id, ".")[1])
+	var country string
+	country = attemptGetCountry(stream.Id, stream.Name)
+	if country == "" {
+		country = attemptGetCountry(stream.meta.originalId, stream.meta.originalName)
+	}
+	return country
+}
+
+func attemptGetCountry(id, name string) string {
+	if id != "" && strings.Count(id, ".") == 1 {
+		return strings.ToUpper(strings.Split(id, ".")[1])
 	}
 
 	regex := `(?i)\b(` + countries + `)\b`
 	r := regexp.MustCompile(regex)
-	matches := r.FindStringSubmatch(stream.Name)
+	matches := r.FindStringSubmatch(name)
 	if matches != nil {
 		country := strings.ToUpper(matches[0])
 
@@ -133,9 +143,19 @@ func findCountry(stream *Stream) string {
 }
 
 func findDefinition(stream *Stream) string {
+	var definition string
+	definition = attemptFindDefinition(stream.Name)
+	if definition == "" {
+		definition = attemptFindDefinition(stream.meta.originalName)
+	}
+
+	return definition
+}
+
+func attemptFindDefinition(name string) string {
 	regex := `(?i)\b(` + definitions + `)\b`
 	r := regexp.MustCompile(regex)
-	matches := r.FindStringSubmatch(stream.Name)
+	matches := r.FindStringSubmatch(name)
 	if matches != nil {
 		definition := strings.ToUpper(matches[0])
 		if val, ok := definitionOverrides[definition]; ok {
@@ -153,13 +173,14 @@ func canonicaliseName(name string) string {
 	name = strings.Replace(name, "|", "", -1)
 	name = regexWordCallback(name, countries, removeWord)
 	name = regexWordCallback(name, definitions, removeWord)
-	name = regexWordCallback(name, "TV", removeWord)
+	name = regexWordCallback(name, stopWords, removeWord)
 	// todo this still isn't correct
 	//if !cache.Regexp("(?i)^Channel \\d+$").Match([]byte(name)) {
 	//	name = regexWordCallback(name, "Channel", removeWord)
 	//}
 
-	name = strings.Title(name)
+	name = regexWordCallback(name, " +", func(s string) string { return " " })
 	name = strings.ToLower(name)
+	name = strings.Title(name)
 	return strings.TrimSpace(name)
 }