From 21e0533f9191ed5503a95983ae24c63d8cff37f2 Mon Sep 17 00:00:00 2001 From: Dan Lenski Date: Fri, 20 Feb 2015 00:32:59 -0800 Subject: [PATCH] add options to ignore errors (-E) and to export specific formats (-f), and make incremental_backup.py more intelligent about finding which activities have already been backed up in the desired formats --- garminexport.py | 26 +++++++++++----- garminexport/garminclient.py | 17 ++++++---- garminexport/util.py | 2 +- incremental_backup.py | 60 +++++++++++++++++++++--------------- samples/sample.py | 4 +-- 5 files changed, 69 insertions(+), 40 deletions(-) diff --git a/garminexport.py b/garminexport.py index e51fe60..fe01946 100755 --- a/garminexport.py +++ b/garminexport.py @@ -42,6 +42,14 @@ if __name__ == "__main__": "--log-level", metavar="LEVEL", type=str, help=("Desired log output level (DEBUG, INFO, WARNING, ERROR). " "Default: INFO."), default="INFO") + parser.add_argument( + "-f", "--format", choices=garminexport.util.export_formats, + default=None, action='append', + help=("Desired output formats ("+', '.join(garminexport.util.export_formats)+"). " + "Default: ALL.")) + parser.add_argument( + "-E", "--ignore-errors", action='store_true', + help="Ignore errors and keep going. Default: FALSE") args = parser.parse_args() if not args.log_level in LOG_LEVELS: @@ -57,14 +65,18 @@ if __name__ == "__main__": with GarminClient(args.username, args.password) as client: log.info("fetching activities for {} ...".format(args.username)) - activity_ids = client.list_activity_ids() - for index, id in enumerate(activity_ids): - log.info("processing activity {} ({} out of {}) ...".format( - id, index+1, len(activity_ids))) - garminexport.util.export_activity( - client, id, args.destination) + all_activities = client.list_activities() + for index, (id, start) in enumerate(activity_ids): + log.info("processing activity {} from {} ({} out of {}) ...".format( + id, start, index+1, len(activity_ids))) + try: + garminexport.util.export_activity( + client, id, args.backup_dir, args.format) + except Exception as e: + log.error(u"failed with exception: %s", e) + if not args.ignore_errors: + raise except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() log.error(u"failed with exception: %s", e) raise - diff --git a/garminexport/garminclient.py b/garminexport/garminclient.py index 9daef0c..46fd49e 100755 --- a/garminexport/garminclient.py +++ b/garminexport/garminclient.py @@ -10,6 +10,7 @@ import requests from StringIO import StringIO import sys import zipfile +import dateutil # # Note: For more detailed information about the API services @@ -147,25 +148,26 @@ class GarminClient(object): @require_session - def list_activity_ids(self): - """Return all activity ids stored by the logged in user. + def list_activities(self): + """Return all activity ids stored by the logged in user, along + with their starting timestamps. :returns: The full list of activity identifiers. - :rtype: list of str + :rtype: tuples of (int, datetime) """ ids = [] batch_size = 100 # fetch in batches since the API doesn't allow more than a certain # number of activities to be retrieved on every invocation for start_index in xrange(0, sys.maxint, batch_size): - next_batch = self._fetch_activity_ids(start_index, batch_size) + next_batch = self._fetch_activity_ids_and_ts(start_index, batch_size) if not next_batch: break ids.extend(next_batch) return ids @require_session - def _fetch_activity_ids(self, start_index, max_limit=100): + def _fetch_activity_ids_and_ts(self, start_index, max_limit=100): """Return a sequence of activity ids starting at a given index, with index 0 being the user's most recently registered activity. @@ -193,7 +195,10 @@ class GarminClient(object): if not "activities" in results: # index out of bounds or empty account return [] - entries = [int(entry["activity"]["activityId"]) for entry in results["activities"]] + + entries = [ (int(entry["activity"]["activityId"]), + dateutil.parser.parse(entry["activity"]["activitySummary"]["BeginTimestamp"]["value"])) + for entry in results["activities"] ] log.debug("got {} activities.".format(len(entries))) return entries diff --git a/garminexport/util.py b/garminexport/util.py index 857bf61..e4ee75e 100644 --- a/garminexport/util.py +++ b/garminexport/util.py @@ -28,7 +28,7 @@ def export_activity(client, activity_id, destination, :type formats: list of str """ if formats is None: - formats = ['json_summary', 'json_details', 'gpx', 'tcx', 'fit'] + formats = export_formats activity_summary = client.get_activity_summary(activity_id) # prefix saved activity files with timestamp and activity id diff --git a/incremental_backup.py b/incremental_backup.py index 5ef9a88..b0c7db9 100755 --- a/incremental_backup.py +++ b/incremental_backup.py @@ -27,24 +27,24 @@ LOG_LEVELS = { } """Command-line (string-based) log-level mapping to logging module levels.""" -def get_backed_up_ids(backup_dir): - """Return all activitiy ids that have been backed up in the +def get_backed_up(activities, backup_dir, formats): + """Return all activity (id, ts) pairs that have been backed up in the given backup directory. :rtype: list of int """ # backed up activities follow this pattern: __summary.json - activity_file_pattern = r'[\d:T\+\-]+_([0-9]+)_summary\.json' + activity_file_pattern = r'[\d:T\+\-]+_([0-9]+).tcx' + + format_suffix = dict(json_summary="_summary.json", json_details="_details.json", gpx=".gpx", tcx=".tcx", fit=".fit") - backed_up_ids = [] + backed_up = set() dir_entries = os.listdir(backup_dir) - for entry in dir_entries: - activity_match = re.search(activity_file_pattern, entry) - if activity_match: - backed_up_id = int(activity_match.group(1)) - backed_up_ids.append(backed_up_id) - return backed_up_ids - + for id, start in activities: + if all( "{}_{}{}".format(start.isoformat(), id, format_suffix[f]) in dir_entries for f in formats): + backed_up.add((id, start)) + return backed_up + if __name__ == "__main__": parser = argparse.ArgumentParser( @@ -67,6 +67,14 @@ if __name__ == "__main__": "--log-level", metavar="LEVEL", type=str, help=("Desired log output level (DEBUG, INFO, WARNING, ERROR). " "Default: INFO."), default="INFO") + parser.add_argument( + "-f", "--format", choices=garminexport.util.export_formats, + default=None, action='append', + help=("Desired output formats ("+', '.join(garminexport.util.export_formats)+"). " + "Default: ALL.")) + parser.add_argument( + "-E", "--ignore-errors", action='store_true', + help="Ignore errors and keep going. Default: FALSE") args = parser.parse_args() if not args.log_level in LOG_LEVELS: @@ -81,28 +89,32 @@ if __name__ == "__main__": args.password = getpass.getpass("Enter password: ") with GarminClient(args.username, args.password) as client: - # already backed up activities (stored in backup-dir) - backed_up_activities = set(get_backed_up_ids(args.backup_dir)) - log.info("{} contains {} backed up activities.".format( - args.backup_dir, len(backed_up_activities))) - - # get all activity ids from Garmin account + # get all activity ids and timestamps from Garmin account log.info("retrieving activities for {} ...".format(args.username)) - all_activities = set(client.list_activity_ids()) + all_activities = set(client.list_activities()) log.info("account has a total of {} activities.".format( len(all_activities))) + # get already backed up activities (stored in backup-dir) + backed_up_activities = get_backed_up(all_activities, args.backup_dir, args.format) + log.info("{} contains {} backed up activities.".format( + args.backup_dir, len(backed_up_activities))) + missing_activities = all_activities - backed_up_activities log.info("activities that haven't been backed up: {}".format( len(missing_activities))) - for index, id in enumerate(missing_activities): - log.info("backing up activity {} ({} out of {}) ...".format( - id, index+1, len(missing_activities))) - garminexport.util.export_activity( - client, id, args.backup_dir) + for index, (id, start) in enumerate(missing_activities): + log.info("backing up activity {} from {} ({} out of {}) ...".format( + id, start, index+1, len(missing_activities))) + try: + garminexport.util.export_activity( + client, id, args.backup_dir, args.format) + except Exception as e: + log.error(u"failed with exception: %s", e) + if not args.ignore_errors: + raise except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() log.error(u"failed with exception: %s", e) raise - diff --git a/samples/sample.py b/samples/sample.py index 17e7210..9438802 100755 --- a/samples/sample.py +++ b/samples/sample.py @@ -30,11 +30,11 @@ if __name__ == "__main__": try: with GarminClient(args.username, args.password) as client: log.info("activities:") - activity_ids = client.list_activity_ids() + activity_ids = client.list_activities() log.info("num ids: {}".format(len(activity_ids))) log.info(activity_ids) - latest_activity = activity_ids[0] + latest_activity, latest_activity_start = activity_ids[0] activity = client.get_activity_summary(latest_activity) log.info(u"activity id: %s", activity["activity"]["activityId"]) log.info(u"activity name: '%s'", activity["activity"]["activityName"])