add options to ignore errors (-E) and to export specific formats (-f), and make incremental_backup.py more intelligent about finding which activities have already been backed up in the desired formats

garminexport.py

@@ -42,6 +42,14 @@ if __name__ == "__main__":
         "--log-level", metavar="LEVEL", type=str,
         help=("Desired log output level (DEBUG, INFO, WARNING, ERROR). "
               "Default: INFO."), default="INFO")
+    parser.add_argument(
+        "-f", "--format", choices=garminexport.util.export_formats,
+        default=None, action='append',
+        help=("Desired output formats ("+', '.join(garminexport.util.export_formats)+"). "
+              "Default: ALL."))
+    parser.add_argument(
+        "-E", "--ignore-errors", action='store_true',
+        help="Ignore errors and keep going. Default: FALSE")
     
     args = parser.parse_args()
     if not args.log_level in LOG_LEVELS:
@@ -57,14 +65,18 @@ if __name__ == "__main__":
         
         with GarminClient(args.username, args.password) as client:
             log.info("fetching activities for {} ...".format(args.username))
-            activity_ids = client.list_activity_ids()
+            all_activities = client.list_activities()
-            for index, id in enumerate(activity_ids):
+            for index, (id, start) in enumerate(activity_ids):
-                log.info("processing activity {} ({} out of {}) ...".format(
+                log.info("processing activity {} from {} ({} out of {}) ...".format(
-                    id, index+1, len(activity_ids)))
+                    id, start, index+1, len(activity_ids)))
-                garminexport.util.export_activity(
+                try:
-                    client, id, args.destination)
+                    garminexport.util.export_activity(
+                        client, id, args.backup_dir, args.format)
+                except Exception as e:
+                    log.error(u"failed with exception: %s", e)
+                    if not args.ignore_errors:
+                        raise
     except Exception as e:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         log.error(u"failed with exception: %s", e)
         raise
-

garminexport/garminclient.py

@@ -10,6 +10,7 @@ import requests
 from StringIO import StringIO
 import sys
 import zipfile
+import dateutil
 
 #
 # Note: For more detailed information about the API services
@@ -147,25 +148,26 @@ class GarminClient(object):
         
         
     @require_session
-    def list_activity_ids(self):
+    def list_activities(self):
-        """Return all activity ids stored by the logged in user.
+        """Return all activity ids stored by the logged in user, along
+        with their starting timestamps.
 
         :returns: The full list of activity identifiers.
-        :rtype: list of str
+        :rtype: tuples of (int, datetime)
         """        
         ids = []
         batch_size = 100
         # fetch in batches since the API doesn't allow more than a certain
         # number of activities to be retrieved on every invocation
         for start_index in xrange(0, sys.maxint, batch_size):
-            next_batch = self._fetch_activity_ids(start_index, batch_size)
+            next_batch = self._fetch_activity_ids_and_ts(start_index, batch_size)
             if not next_batch:
                 break
             ids.extend(next_batch)
         return ids
 
     @require_session
-    def _fetch_activity_ids(self, start_index, max_limit=100):
+    def _fetch_activity_ids_and_ts(self, start_index, max_limit=100):
         """Return a sequence of activity ids starting at a given index,
         with index 0 being the user's most recently registered activity.
 
@@ -193,7 +195,10 @@ class GarminClient(object):
         if not "activities" in results:
             # index out of bounds or empty account
             return []
-        entries = [int(entry["activity"]["activityId"]) for entry in results["activities"]]
+
+        entries = [ (int(entry["activity"]["activityId"]),
+                     dateutil.parser.parse(entry["activity"]["activitySummary"]["BeginTimestamp"]["value"]))
+                    for entry in results["activities"] ]
         log.debug("got {} activities.".format(len(entries)))
         return entries
                

garminexport/util.py

@@ -28,7 +28,7 @@ def export_activity(client, activity_id, destination,
     :type formats: list of str
     """
     if formats is None:
-        formats = ['json_summary', 'json_details', 'gpx', 'tcx', 'fit']
+        formats = export_formats
     activity_summary = client.get_activity_summary(activity_id)
     
     # prefix saved activity files with timestamp and activity id

incremental_backup.py

@@ -27,23 +27,23 @@ LOG_LEVELS = {
 }
 """Command-line (string-based) log-level mapping to logging module levels."""
 
-def get_backed_up_ids(backup_dir):
+def get_backed_up(activities, backup_dir, formats):
-    """Return all activitiy ids that have been backed up in the
+    """Return all activity (id, ts) pairs that have been backed up in the
     given backup directory.
 
     :rtype: list of int
     """
     # backed up activities follow this pattern: <ISO8601>_<id>_summary.json
-    activity_file_pattern = r'[\d:T\+\-]+_([0-9]+)_summary\.json'
+    activity_file_pattern = r'[\d:T\+\-]+_([0-9]+).tcx'
 
-    backed_up_ids = []
+    format_suffix = dict(json_summary="_summary.json", json_details="_details.json", gpx=".gpx", tcx=".tcx", fit=".fit")
+    
+    backed_up = set()
     dir_entries = os.listdir(backup_dir)
-    for entry in dir_entries:
+    for id, start in activities:
-        activity_match = re.search(activity_file_pattern, entry)
+        if all( "{}_{}{}".format(start.isoformat(), id, format_suffix[f]) in dir_entries for f in formats):
-        if activity_match:
+            backed_up.add((id, start))
-            backed_up_id = int(activity_match.group(1))
+    return backed_up
-            backed_up_ids.append(backed_up_id)
-    return backed_up_ids           
 
 
 if __name__ == "__main__":
@@ -67,6 +67,14 @@ if __name__ == "__main__":
         "--log-level", metavar="LEVEL", type=str,
         help=("Desired log output level (DEBUG, INFO, WARNING, ERROR). "
               "Default: INFO."), default="INFO")
+    parser.add_argument(
+        "-f", "--format", choices=garminexport.util.export_formats,
+        default=None, action='append',
+        help=("Desired output formats ("+', '.join(garminexport.util.export_formats)+"). "
+              "Default: ALL."))
+    parser.add_argument(
+        "-E", "--ignore-errors", action='store_true',
+        help="Ignore errors and keep going. Default: FALSE")
     
     args = parser.parse_args()
     if not args.log_level in LOG_LEVELS:
@@ -81,28 +89,32 @@ if __name__ == "__main__":
             args.password = getpass.getpass("Enter password: ")
         
         with GarminClient(args.username, args.password) as client:
-            # already backed up activities (stored in backup-dir)
+            # get all activity ids and timestamps from Garmin account
-            backed_up_activities = set(get_backed_up_ids(args.backup_dir))
-            log.info("{} contains {} backed up activities.".format(
-                args.backup_dir, len(backed_up_activities)))
-
-            # get all activity ids from Garmin account
             log.info("retrieving activities for {} ...".format(args.username))
-            all_activities = set(client.list_activity_ids())
+            all_activities = set(client.list_activities())
             log.info("account has a total of {} activities.".format(
                 len(all_activities)))
             
+            # get already backed up activities (stored in backup-dir)
+            backed_up_activities = get_backed_up(all_activities, args.backup_dir, args.format)
+            log.info("{} contains {} backed up activities.".format(
+                args.backup_dir, len(backed_up_activities)))
+
             missing_activities = all_activities - backed_up_activities
             log.info("activities that haven't been backed up: {}".format(
                 len(missing_activities)))
             
-            for index, id in enumerate(missing_activities):
+            for index, (id, start) in enumerate(missing_activities):
-                log.info("backing up activity {} ({} out of {}) ...".format(
+                log.info("backing up activity {} from {} ({} out of {}) ...".format(
-                    id, index+1, len(missing_activities)))
+                    id, start, index+1, len(missing_activities)))
-                garminexport.util.export_activity(
+                try:
-                    client, id, args.backup_dir)
+                    garminexport.util.export_activity(
+                        client, id, args.backup_dir, args.format)
+                except Exception as e:
+                    log.error(u"failed with exception: %s", e)
+                    if not args.ignore_errors:
+                        raise
     except Exception as e:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         log.error(u"failed with exception: %s", e)
         raise
-

samples/sample.py

@@ -30,11 +30,11 @@ if __name__ == "__main__":
     try:
         with GarminClient(args.username, args.password) as client:
             log.info("activities:")
-            activity_ids = client.list_activity_ids()
+            activity_ids = client.list_activities()
             log.info("num ids: {}".format(len(activity_ids)))
             log.info(activity_ids)
 
-            latest_activity = activity_ids[0]
+            latest_activity, latest_activity_start = activity_ids[0]
             activity = client.get_activity_summary(latest_activity)
             log.info(u"activity id: %s", activity["activity"]["activityId"])
             log.info(u"activity name: '%s'", activity["activity"]["activityName"])