From c07f9b5f72bd22bff9c4e53ff330448f7fabae2a Mon Sep 17 00:00:00 2001 From: Jonathan Kamens Date: Tue, 28 Mar 2017 10:38:38 -0400 Subject: [PATCH 1/9] Use argparse for parsing command line In preparation for adding a configuration file and command-line arguments, add code to use argparse to parse the command line. This is at this point nonfunctional with the exception that the code now enforces exactly the correct number of arguments on the command line rather than silently ignoring extra arguments. --- ExportLists.py | 17 ++++++++++++----- ImportList.py | 15 +++++++++++---- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/ExportLists.py b/ExportLists.py index 1b7e506..2a0f4c4 100644 --- a/ExportLists.py +++ b/ExportLists.py @@ -1,15 +1,22 @@ # Author: John Elkins # License: MIT +import argparse +import os from common import * -if len(sys.argv) < 2: - log('ERROR output directory is required') - time.sleep(3) - exit() +def parse_args(): + parser = argparse.ArgumentParser(description="Export playlists from " + "Google Play Music") + parser.add_argument("output-directory", action="store", + help="Directory into which to export playlists") + args = parser.parse_args() + return args + +args = parse_args() # setup the output directory, create it if needed -output_dir = sys.argv[1] +output_dir = args.output_directory if not os.path.exists(output_dir): os.makedirs(output_dir) diff --git a/ImportList.py b/ImportList.py index 1d37392..9409e30 100644 --- a/ImportList.py +++ b/ImportList.py @@ -1,6 +1,8 @@ # Author: John Elkins # License: MIT +import argparse +import os import re import datetime import math @@ -162,13 +164,18 @@ def score_track(details,result_details,top_score = 200): return (result_score,score_reason) -# check to make sure a filename was given -if len(sys.argv) < 2: - delayed_exit(u'ERROR input filename is required') +def parse_args(): + parser = argparse.ArgumentParser(description="Import playlist into Google " + "Play Music") + parser.add_argument("playlist-filename", action="store", + help="Playlist CSV file to import") + args = parser.parse_args() + return args +args = parse_args() # setup the input and output filenames and derive the playlist name -input_filename = sys.argv[1].decode('utf-8') +input_filename = args.playlist_filename.decode('utf-8') output_filename = os.path.splitext(input_filename)[0] output_filename = re.compile('_\d{14}$').sub(u'',output_filename) playlist_name = os.path.basename(output_filename) From 12430ea9167355900417e2911dbbac05ad564f11 Mon Sep 17 00:00:00 2001 From: Jonathan Kamens Date: Tue, 28 Mar 2017 10:43:41 -0400 Subject: [PATCH 2/9] Specify username on command line instead of hard-coding --- ExportLists.py | 7 +++++-- ImportList.py | 7 +++++-- README.md | 2 -- common.py | 2 +- preferences.py | 4 ---- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ExportLists.py b/ExportLists.py index 2a0f4c4..c2a58fd 100644 --- a/ExportLists.py +++ b/ExportLists.py @@ -8,7 +8,10 @@ def parse_args(): parser = argparse.ArgumentParser(description="Export playlists from " "Google Play Music") - parser.add_argument("output-directory", action="store", + parser.add_argument("--username", action="store", help="Google username", + required=True) + parser.add_argument("output_directory", action="store", + metavar="output-directory", help="Directory into which to export playlists") args = parser.parse_args() return args @@ -21,7 +24,7 @@ def parse_args(): os.makedirs(output_dir) # log in and load personal library -api = open_api() +api = open_api(args.username) library = load_personal_library() def playlist_handler(playlist_name, playlist_description, playlist_tracks): diff --git a/ImportList.py b/ImportList.py index 9409e30..6ff7a2c 100644 --- a/ImportList.py +++ b/ImportList.py @@ -167,7 +167,10 @@ def score_track(details,result_details,top_score = 200): def parse_args(): parser = argparse.ArgumentParser(description="Import playlist into Google " "Play Music") - parser.add_argument("playlist-filename", action="store", + parser.add_argument("--username", action="store", help="Google username", + required=True) + parser.add_argument("playlist_filename", action="store", + metavar="playlist-filename", help="Playlist CSV file to import") args = parser.parse_args() return args @@ -197,7 +200,7 @@ def parse_args(): log('done. '+str(len(tracks))+' lines loaded.') # log in and load personal library -api = open_api() +api = open_api(args.username) library = load_personal_library() # begin searching for the tracks diff --git a/README.md b/README.md index 2836927..5e666d6 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,6 @@ playlist scripts for gmusic - python 2.7 - https://www.python.org - gmusicapi - https://github.com/simon-weber/Unofficial-Google-Music-API -Before using the scripts, open up the preferences.py file and change the username. - When the scripts are run they will prompt for your password. If you use two factor authentication you will need to create and use an application password. ## ExportLists.py diff --git a/common.py b/common.py index 30473ca..e31329f 100644 --- a/common.py +++ b/common.py @@ -171,7 +171,7 @@ def create_details_string(details_dict, skip_id = False): return out_string # logs into google music api -def open_api(): +def open_api(username): global api log('Logging into google music...') # get the password each time so that it isn't stored in plain text diff --git a/preferences.py b/preferences.py index af5a2b5..664dd3d 100644 --- a/preferences.py +++ b/preferences.py @@ -1,7 +1,3 @@ - -# the username to use -username = 'john.elkins@gmail.com' - # the separator to use for detailed track information track_info_separator = u',' #track_info_separator = u'\\' From 3a9312571213dde9f8f417bdc3bf66b332b29a88 Mon Sep 17 00:00:00 2001 From: Jonathan Kamens Date: Tue, 28 Mar 2017 11:04:48 -0400 Subject: [PATCH 3/9] Allow username to be specified in INI file --- ExportLists.py | 16 +++------------- ImportList.py | 16 +++------------- common.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/ExportLists.py b/ExportLists.py index c2a58fd..2b2ea3f 100644 --- a/ExportLists.py +++ b/ExportLists.py @@ -1,22 +1,12 @@ # Author: John Elkins # License: MIT -import argparse import os from common import * -def parse_args(): - parser = argparse.ArgumentParser(description="Export playlists from " - "Google Play Music") - parser.add_argument("--username", action="store", help="Google username", - required=True) - parser.add_argument("output_directory", action="store", - metavar="output-directory", - help="Directory into which to export playlists") - args = parser.parse_args() - return args - -args = parse_args() +args = parse_args('Export playlists from Google Play Music', + 'ExportLists', 'output_directory', + 'Directory into which to export playlists') # setup the output directory, create it if needed output_dir = args.output_directory diff --git a/ImportList.py b/ImportList.py index 6ff7a2c..4580bf2 100644 --- a/ImportList.py +++ b/ImportList.py @@ -1,7 +1,6 @@ # Author: John Elkins # License: MIT -import argparse import os import re import datetime @@ -164,18 +163,9 @@ def score_track(details,result_details,top_score = 200): return (result_score,score_reason) -def parse_args(): - parser = argparse.ArgumentParser(description="Import playlist into Google " - "Play Music") - parser.add_argument("--username", action="store", help="Google username", - required=True) - parser.add_argument("playlist_filename", action="store", - metavar="playlist-filename", - help="Playlist CSV file to import") - args = parser.parse_args() - return args - -args = parse_args() +args = parse_args('Import playlist into Google Play Music', + 'ImportList', 'playlist_filename', + 'Playlist CSV file to import') # setup the input and output filenames and derive the playlist name input_filename = args.playlist_filename.decode('utf-8') diff --git a/common.py b/common.py index e31329f..e4a0f41 100644 --- a/common.py +++ b/common.py @@ -5,6 +5,8 @@ __required_gmusicapi_version__ = '10.0.0' +import argparse +from ConfigParser import SafeConfigParser from collections import Counter from gmusicapi import __version__ as gmusicapi_version from gmusicapi import Mobileclient @@ -33,6 +35,46 @@ if '-dDEBUG' in sys.argv: debug = True +def parse_args(description, program_name, path_target, path_help): + default_cf = os.path.expanduser("~/.gmusic-playlist.ini") + parser = argparse.ArgumentParser(description=description) + parser.add_argument("--config-file", "-cf", action="store", + help="INI file to read settings from (default {})". + format(default_cf), default=default_cf) + parser.add_argument("--username", action="store", help="Google username") + parser.add_argument(path_target, action="store", + metavar=path_target.replace('_', '-'), + help=path_help) + args = parser.parse_args() + + if not args.username: + args.username = get_config_username(program_name, args.config_file) + return args + +def read_config_file(filename): + parser = SafeConfigParser() + parser.read(filename) + return parser + +def get_config_setting(program_name, config_file, setting): + parser = read_config_file(config_file) + try: + value = parser.get(program_name, setting) + return value + except: + try: + value = parser.get('defaults', setting) + return value + except: + return None + +def get_config_username(program_name, config_file): + username = get_config_setting(program_name, config_file, 'username') + if not username: + sys.exit('Specify username on command line or put it in "{}" or ' + '"defaults" section of {}'.format(program_name, config_file)) + return username + # check versions def assert_prerequisites(): From 52bbba571ecc159a79223a95eb8b7fc93c46abfd Mon Sep 17 00:00:00 2001 From: Jonathan Kamens Date: Tue, 28 Mar 2017 11:26:16 -0400 Subject: [PATCH 4/9] Make Android ID parameterized Allow the Android ID to be specified in the configuration file or on the command line, or randomly generated and saved in the configuration file automatically. --- ExportLists.py | 2 +- ImportList.py | 2 +- common.py | 32 ++++++++++++++++++++++++++++++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/ExportLists.py b/ExportLists.py index 2b2ea3f..1fde48b 100644 --- a/ExportLists.py +++ b/ExportLists.py @@ -14,7 +14,7 @@ os.makedirs(output_dir) # log in and load personal library -api = open_api(args.username) +api = open_api(args.username, args.android_id) library = load_personal_library() def playlist_handler(playlist_name, playlist_description, playlist_tracks): diff --git a/ImportList.py b/ImportList.py index 4580bf2..226f0cb 100644 --- a/ImportList.py +++ b/ImportList.py @@ -190,7 +190,7 @@ def score_track(details,result_details,top_score = 200): log('done. '+str(len(tracks))+' lines loaded.') # log in and load personal library -api = open_api(args.username) +api = open_api(args.username, args.android_id) library = load_personal_library() # begin searching for the tracks diff --git a/common.py b/common.py index e4a0f41..979fab6 100644 --- a/common.py +++ b/common.py @@ -12,6 +12,7 @@ from gmusicapi import Mobileclient from gmusicapi.exceptions import CallFailure from preferences import * +import random import re import time import getpass @@ -42,6 +43,12 @@ def parse_args(description, program_name, path_target, path_help): help="INI file to read settings from (default {})". format(default_cf), default=default_cf) parser.add_argument("--username", action="store", help="Google username") + group = parser.add_mutually_exclusive_group() + group.add_argument("--android-id", action="store", help="Android ID to " + "connect with instead of using MAC address") + group.add_argument("--create-android-id", action="store_true", + help="Generate a random android ID and store it in the " + "configuration file for future use") parser.add_argument(path_target, action="store", metavar=path_target.replace('_', '-'), help=path_help) @@ -49,6 +56,9 @@ def parse_args(description, program_name, path_target, path_help): if not args.username: args.username = get_config_username(program_name, args.config_file) + if not args.android_id or args.create_android_id: + args.android_id = get_config_android_id(program_name, args.config_file, + args.create_android_id) return args def read_config_file(filename): @@ -75,6 +85,23 @@ def get_config_username(program_name, config_file): '"defaults" section of {}'.format(program_name, config_file)) return username +def get_config_android_id(program_name, config_file, do_create): + if do_create: + parser = read_config_file(config_file) + android_id = ''.join(hex(int(random.random() * 16))[-1] for + i in range(16)) + if not parser.has_section('defaults'): + parser.add_section('defaults') + parser.set('defaults', 'android_id', android_id) + parser.write(open(config_file, 'w')) + return android_id + android_id = get_config_setting(program_name, config_file, 'android_id') + if not android_id: + sys.exit('Specify android ID or --create-android-id on command line ' + 'or put it in "{}" or "defaults" section of {}'.format( + program_name, config_file)) + return android_id + # check versions def assert_prerequisites(): @@ -213,14 +240,15 @@ def create_details_string(details_dict, skip_id = False): return out_string # logs into google music api -def open_api(username): +def open_api(username, android_id=None): global api log('Logging into google music...') # get the password each time so that it isn't stored in plain text password = getpass.getpass(username + '\'s password: ') api = Mobileclient() - if not api.login(username, password, Mobileclient.FROM_MAC_ADDRESS): + if not api.login(username, password, + android_id or Mobileclient.FROM_MAC_ADDRESS): log('ERROR unable to login') time.sleep(3) exit() From 156a8158557214dbcfd80166b38af873e9797e02 Mon Sep 17 00:00:00 2001 From: Jonathan Kamens Date: Tue, 28 Mar 2017 11:53:10 -0400 Subject: [PATCH 5/9] Allow playlists to be replaced on import --- ImportList.py | 22 +++++++++++++++++++++- common.py | 6 +++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/ImportList.py b/ImportList.py index 226f0cb..3876304 100644 --- a/ImportList.py +++ b/ImportList.py @@ -165,7 +165,12 @@ def score_track(details,result_details,top_score = 200): args = parse_args('Import playlist into Google Play Music', 'ImportList', 'playlist_filename', - 'Playlist CSV file to import') + 'Playlist CSV file to import', + extra_args=( + (('--replace',), + {'action': 'store_true', + 'help': 'Replace playlist(s) with the same name'}), + )) # setup the input and output filenames and derive the playlist name input_filename = args.playlist_filename.decode('utf-8') @@ -295,6 +300,21 @@ def score_track(details,result_details,top_score = 200): log(u'Adding '+unicode(len(song_ids))+' found songs to: '+playlist_name) log('===============================================================') +# delete old playlists if requested +if args.replace: + for playlist in api.get_all_playlists(): + if playlist['kind'] != 'sj#playlist': + continue + if playlist['type'] != 'USER_GENERATED': + continue + if (playlist['name'] != playlist_name and + not playlist['name'].startswith(u'{} Part '.format( + playlist_name))): + continue + api.delete_playlist(playlist['id']) + log(u'Deleted playlist {} named {}'.format( + playlist['id'], playlist['name'])) + # add the songs to the playlist(s) max_playlist_size = 1000 current_playlist = 1 diff --git a/common.py b/common.py index 979fab6..2645fc4 100644 --- a/common.py +++ b/common.py @@ -36,7 +36,8 @@ if '-dDEBUG' in sys.argv: debug = True -def parse_args(description, program_name, path_target, path_help): +def parse_args(description, program_name, path_target, path_help, + extra_args=()): default_cf = os.path.expanduser("~/.gmusic-playlist.ini") parser = argparse.ArgumentParser(description=description) parser.add_argument("--config-file", "-cf", action="store", @@ -52,6 +53,9 @@ def parse_args(description, program_name, path_target, path_help): parser.add_argument(path_target, action="store", metavar=path_target.replace('_', '-'), help=path_help) + for arg in extra_args: + parser.add_argument(*arg[0], **arg[1]) + args = parser.parse_args() if not args.username: From 583f17cad46a95662c61144cd471ce2f02df40a1 Mon Sep 17 00:00:00 2001 From: Jonathan Kamens Date: Tue, 28 Mar 2017 12:10:38 -0400 Subject: [PATCH 6/9] Don't use storeId The code was using the storeId field in songs to import them, when it existed. I don't know why it was doing that, but it wasn't working for me -- songs added via storeId simply weren't appearing in my playlists. Removing that logic and always using the id, never the storeId, fixed this problem. This change may not be "correct", but all I know is that the code wasn't working for me without the change and is working with it. --- common.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/common.py b/common.py index 2645fc4..c2c6a52 100644 --- a/common.py +++ b/common.py @@ -180,8 +180,16 @@ def create_result_details(track): result_details = {} for key, value in track.iteritems(): result_details[key] = value - result_details['songid'] = (track.get('storeId') - if track.get('storeId') else track.get('id')) +# Previously, the code did this: +# +# result_details['songid'] = (track.get('storeId') +# if track.get('storeId') else track.get('id')) +# +# But when the code was doing that, songs which had a storeId were not +# being added to my playlists. I do not understand why storeId was +# being used instead of id; all I know is that removing that logic +# fixes my playlists. + result_details['songid'] = track.get('id') return result_details # creates details dictionary based off the given details list From 993c10f534df6006ccff2669ba3448e6e64a149b Mon Sep 17 00:00:00 2001 From: Jonathan Kamens Date: Tue, 28 Mar 2017 12:17:52 -0400 Subject: [PATCH 7/9] Allow password to be specified in configuration file --- ExportLists.py | 2 +- ImportList.py | 2 +- common.py | 10 +++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ExportLists.py b/ExportLists.py index 1fde48b..4665ff7 100644 --- a/ExportLists.py +++ b/ExportLists.py @@ -14,7 +14,7 @@ os.makedirs(output_dir) # log in and load personal library -api = open_api(args.username, args.android_id) +api = open_api(args.username, args.android_id, password=args.password) library = load_personal_library() def playlist_handler(playlist_name, playlist_description, playlist_tracks): diff --git a/ImportList.py b/ImportList.py index 3876304..a04bdab 100644 --- a/ImportList.py +++ b/ImportList.py @@ -195,7 +195,7 @@ def score_track(details,result_details,top_score = 200): log('done. '+str(len(tracks))+' lines loaded.') # log in and load personal library -api = open_api(args.username, args.android_id) +api = open_api(args.username, args.android_id, password=args.password) library = load_personal_library() # begin searching for the tracks diff --git a/common.py b/common.py index c2c6a52..844bdc0 100644 --- a/common.py +++ b/common.py @@ -60,6 +60,7 @@ def parse_args(description, program_name, path_target, path_help, if not args.username: args.username = get_config_username(program_name, args.config_file) + args.password = get_config_password(program_name, args.config_file) if not args.android_id or args.create_android_id: args.android_id = get_config_android_id(program_name, args.config_file, args.create_android_id) @@ -89,6 +90,9 @@ def get_config_username(program_name, config_file): '"defaults" section of {}'.format(program_name, config_file)) return username +def get_config_password(program_name, config_file): + return get_config_setting(program_name, config_file, 'password') + def get_config_android_id(program_name, config_file, do_create): if do_create: parser = read_config_file(config_file) @@ -252,11 +256,11 @@ def create_details_string(details_dict, skip_id = False): return out_string # logs into google music api -def open_api(username, android_id=None): +def open_api(username, android_id=None, password=None): global api log('Logging into google music...') - # get the password each time so that it isn't stored in plain text - password = getpass.getpass(username + '\'s password: ') + if not password: + password = getpass.getpass(username + '\'s password: ') api = Mobileclient() if not api.login(username, password, From 108cd2b1514efdc2a4d967c4a3aae8681676cbff Mon Sep 17 00:00:00 2001 From: Jonathan Kamens Date: Tue, 28 Mar 2017 14:23:03 -0400 Subject: [PATCH 8/9] Be smarter about picking the best match and dropping duplicates When searching for a track, if there are multiple matches and some of them are already selected, then use the ones that aren't. When there are multiple search results, prefer exact matches rather than substring matches for title, album, artist. --- ImportList.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/ImportList.py b/ImportList.py index a04bdab..f4f93c3 100644 --- a/ImportList.py +++ b/ImportList.py @@ -57,6 +57,34 @@ def log_unmatched(track): csvfile.write(os.linesep) no_matches += 1 +def best_match(details, results): + # Two heuristics for choosing the best match: + # + # 1. If some of the results are already in song_ids, then choose + # one that isn't. + # 2. Prefer an exact match on title, album, artist (in that order) + # over a substring match. + if len(results) == 1: + return results[0] + + # create_result_details determines the songid + pairs = ((r, create_result_details(r['track'])) for r in results) + + new_pairs = [p for p in pairs if p[1]['songid'] not in song_ids] + if len(new_pairs) == 1: + # If exactly one track isn't already selected, use it. + return new_pairs[0][0] + elif len(new_pairs) > 0: + # Only use the prefer-new heuristic if at least one track passes it. + pairs = new_pairs + + for field in ('title', 'album', 'artist'): + for pair in pairs: + if details[field] and details[field] == pair[1][field]: + return pair[0] + + return pairs[0][0] + # search for the song with the given details def search_for_track(details): search_results = [] @@ -94,7 +122,7 @@ def search_for_track(details): if not len(search_results): return None - top_result = search_results[0] + top_result = best_match(details, search_results) # if we have detailed info, perform a detailed search if details['artist'] and details['title']: search_results = [item for item in search_results if @@ -105,7 +133,7 @@ def search_for_track(details): s_in_s(details['album'],item['track']['album'])] dlog('detail search results: '+str(len(search_results))) if len(search_results) != 0: - top_result = search_results[0] + top_result = best_match(details, search_results) return top_result From 17d8ede749c2fc7f9bac5d0c43438186a38cd707 Mon Sep 17 00:00:00 2001 From: Jonathan Kamens Date: Sun, 2 Apr 2017 08:27:12 -0400 Subject: [PATCH 9/9] Fix best_match traceback when there are no matches --- ImportList.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ImportList.py b/ImportList.py index f4f93c3..1f2207c 100644 --- a/ImportList.py +++ b/ImportList.py @@ -83,7 +83,7 @@ def best_match(details, results): if details[field] and details[field] == pair[1][field]: return pair[0] - return pairs[0][0] + return pairs[0][0] if pairs else None # search for the song with the given details def search_for_track(details):