refactoring

README.md

@@ -1,7 +1,15 @@
 garminexport
 ============
-The Garmin Connect activity exporter is a program that downloads all activities 
+The Garmin Connect activity exporter is a program that downloads *all* 
-for a given [Garmin Connect](http://connect.garmin.com/) account and stores them locally on the user's computer.
+activities for a given [Garmin Connect](http://connect.garmin.com/) 
+account and stores them locally on the user's computer.
+
+The directory also contains an ``incremental_backup.py`` program that can be
+used for incremental backups of your account. This script only downloads
+activities that haven't already been downloaded to a certain backup directory.
+It is typically a quicker alternative (except for the first time when all
+activities will need to be downloaded).
+
 
 Prerequisites
 =============
@@ -12,6 +20,7 @@ assumes that you have [Python 2.7](https://www.python.org/download/releases/2.7/
 It also assumes that you have registered an account at 
 [Garmin Connect](http://connect.garmin.com/).
 
+
 Getting started
 ===============
 Create and activate a new virtual environment to create an isolated development
@@ -24,16 +33,17 @@ Install the required dependencies in this virtual environment:
 
     pip install -r requirements.txt
 
-Run the program
+
-===============
+Running the export program
-The program is run as follows (use the ``--help`` flag for a list of
+==========================
+The export program is run as follows (use the ``--help`` flag for a list of
 available options).
 
     ./garminexport.py <username or email>
 
 Once started, the program will prompt you for your account password and then
-log in to your Garmin Connect account to download all activities to a destination
+log in to your Garmin Connect account to download *all* activities to a 
-directory on your machine.
+destination directory on your machine.
 
 For each activity, these files are stored: 
 
@@ -53,6 +63,18 @@ Each activity file is prefixed by its upload timestamp and its
 activity id.
 
 
+Running the incremental backup program
+======================================
+The incremental backup program is run in a similar fashion to the export 
+program (use the ``--help`` flag for a list of available options):
+
+    ./incremental_backup.py --backup-dir=activities <username or email>
+
+In this example, it will only download activities that aren't already in
+the ``activities/`` directory. Note: The incremental backup program saves
+the same files for each activity as the export program (see above).
+
+
 Library import
 ==============
 To install the development version of this library in your local Python 

garminexport.py

@@ -3,15 +3,11 @@
 and stores them locally on the user's computer.
 """
 import argparse
-import codecs
-from datetime import datetime
 import getpass
 from garminexport.garminclient import GarminClient
-import io
+import garminexport.util
-import json
 import logging
 import os
-import shutil
 import sys
 import traceback
 
@@ -53,7 +49,9 @@ if __name__ == "__main__":
     logging.root.setLevel(LOG_LEVELS[args.log_level])
         
     try:
-        os.makedirs(args.destination)
+        if not os.path.isdir(args.destination):
+            os.makedirs(args.destination)
+
         if not args.password:
             args.password = getpass.getpass("Enter password: ")
         
@@ -61,39 +59,10 @@ if __name__ == "__main__":
             log.info("fetching activities for {} ...".format(args.username))
             activity_ids = client.list_activity_ids()
             for index, id in enumerate(activity_ids):
-                log.info("processing activity {} out of {} ...".format(
+                log.info("processing activity {} ({} out of {}) ...".format(
-                    index+1, len(activity_ids)))
+                    id, index+1, len(activity_ids)))
-                activity_summary = client.get_activity_summary(id)
+                garminexport.util.save_activity(
-                activity_details = client.get_activity_details(id)
+                    client, id, args.destination)
-                activity_gpx = client.get_activity_gpx(id)
-                activity_tcx = client.get_activity_tcx(id)
-                activity_fit = client.get_activity_fit(id)
-                
-                # for each activitity save the summary, details and GPX file.
-                creation_millis = activity_summary["activity"]["uploadDate"]["millis"]
-                timestamp = datetime.fromtimestamp(int(creation_millis)/1000.0)
-                filename_prefix = "{}_{}".format(
-                    timestamp.strftime("%Y%m%d-%H%M%S"), id)
-                path_prefix = os.path.join(args.destination, filename_prefix)
-                
-                summary_file = path_prefix + "_summary.json"
-                details_file = path_prefix + "_details.json"
-                gpx_file = path_prefix + ".gpx"
-                tcx_file = path_prefix + ".tcx"
-                fit_file = path_prefix + ".fit"
-                with codecs.open(summary_file, encoding="utf-8", mode="w") as f:
-                    f.write(json.dumps(
-                        activity_summary, ensure_ascii=False, indent=4))
-                with codecs.open(details_file, encoding="utf-8", mode="w") as f:
-                    f.write(json.dumps(
-                        activity_details, ensure_ascii=False, indent=4))
-                with codecs.open(gpx_file, encoding="utf-8", mode="w") as f:
-                    f.write(activity_gpx)
-                with codecs.open(tcx_file, encoding="utf-8", mode="w") as f:
-                    f.write(activity_tcx)
-                if activity_fit:
-                    with open(fit_file, mode="wb") as f:
-                        f.write(activity_fit)
     except Exception as e:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         log.error(u"failed with exception: %s", e)

garminexport/garminclient.py

@@ -180,7 +180,8 @@ class GarminClient(object):
         :returns: A list of activity identifiers.
         :rtype: list of str
         """
-        log.info("fetching activities {} through {} ...".format(start_index, start_index+max_limit-1))
+        log.debug("fetching activities {} through {} ...".format(
+            start_index, start_index+max_limit-1))
         response = self.session.get(
             "https://connect.garmin.com/proxy/activity-search-service-1.2/json/activities", params={"start": start_index, "limit": max_limit})
         if response.status_code != 200:
@@ -292,5 +293,5 @@ class GarminClient(object):
         # fit file returned from server is in a zip archive
         zipped_fit_file = response.content
         zip = zipfile.ZipFile(StringIO(zipped_fit_file), mode="r")
-        # return the "<activity-id>.fit" entry from the zip archive
+        # return the "<activity-activity_id>.fit" entry from the zip archive
         return zip.open(str(activity_id) + ".fit").read()

garminexport/util.py

@@ -0,0 +1,53 @@
+#! /usr/bin/env python
+"""A module with utility functions."""
+
+import codecs
+import json
+from datetime import datetime
+import os
+
+def save_activity(client, activity_id, destination):
+    """Downloads a certain Garmin Connect activity and saves it
+    to a given destination directory.
+
+    :param client: A :class:`garminexport.garminclient.GarminClient`
+      instance that is assumed to be connected.
+    :type client: :class:`garminexport.garminclient.GarminClient`
+    :param activity_id: Activity identifier.
+    :type activity_id: int
+    :param destination: Destination directory (assumed to exist already).
+    :type destination: str
+    
+    """
+    activity_summary = client.get_activity_summary(activity_id)
+    activity_details = client.get_activity_details(activity_id)
+    activity_gpx = client.get_activity_gpx(activity_id)
+    activity_tcx = client.get_activity_tcx(activity_id)
+    activity_fit = client.get_activity_fit(activity_id)
+                
+    # save activitity summary, details and GPX, TCX and FIT file.
+    creation_millis = activity_summary["activity"]["uploadDate"]["millis"]
+    timestamp = datetime.fromtimestamp(int(creation_millis)/1000.0)
+    filename_prefix = "{}_{}".format(
+        timestamp.strftime("%Y%m%d-%H%M%S"), activity_id)
+    path_prefix = os.path.join(destination, filename_prefix)
+    
+    summary_file = path_prefix + "_summary.json"
+    details_file = path_prefix + "_details.json"
+    gpx_file = path_prefix + ".gpx"
+    tcx_file = path_prefix + ".tcx"
+    fit_file = path_prefix + ".fit"
+    with codecs.open(summary_file, encoding="utf-8", mode="w") as f:
+        f.write(json.dumps(
+            activity_summary, ensure_ascii=False, indent=4))
+    with codecs.open(details_file, encoding="utf-8", mode="w") as f:
+        f.write(json.dumps(
+            activity_details, ensure_ascii=False, indent=4))
+    with codecs.open(gpx_file, encoding="utf-8", mode="w") as f:
+        f.write(activity_gpx)
+    with codecs.open(tcx_file, encoding="utf-8", mode="w") as f:
+        f.write(activity_tcx)
+    if activity_fit:
+        with open(fit_file, mode="wb") as f:
+            f.write(activity_fit)
+    

incremental_backup.py

@@ -0,0 +1,108 @@
+#! /usr/bin/env python
+"""Performs (incremental) backups of activities for a given Garmin Connect
+account.
+The activities are stored in a local directory on the user's computer.
+The backups are incremental, meaning that only activities that aren't already
+stored in the backup directory will be downloaded.
+"""
+import argparse
+import getpass
+from garminexport.garminclient import GarminClient
+import garminexport.util
+import logging
+import os
+import re
+import sys
+import traceback
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)-15s [%(levelname)s] %(message)s")
+log = logging.getLogger(__name__)
+
+LOG_LEVELS = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR
+}
+"""Command-line (string-based) log-level mapping to logging module levels."""
+
+def get_backed_up_ids(backup_dir):
+    """Return all activitiy ids that have been backed up in the
+    given backup directory.
+
+    :rtype: list of int
+    """
+    # backed up activities follow this pattern
+    activity_file_pattern = r'[0-9]+\-[0-9]+_([0-9]+)_summary\.json'
+    
+    backed_up_ids = []
+    dir_entries = os.listdir(backup_dir)
+    for entry in dir_entries:
+        activity_match = re.search(activity_file_pattern, entry)
+        if activity_match:
+            backed_up_id = int(activity_match.group(1))
+            backed_up_ids.append(backed_up_id)
+    return backed_up_ids           
+    
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description=(
+            "Performs incremental backups of activities for a "
+            "given Garmin Connect account. Only activities that "
+            "aren't already stored in the backup directory will "
+            "be downloaded."))
+    # positional args
+    parser.add_argument(
+        "username", metavar="<username>", type=str, help="Account user name.")
+    # optional args
+    parser.add_argument(
+        "--password", type=str, help="Account password.")
+    parser.add_argument(
+        "--backup-dir", metavar="DIR", type=str,
+        help=("Destination directory for downloaded activities. Default: "
+              "./activities/"), default=os.path.join(".", "activities"))
+    parser.add_argument(
+        "--log-level", metavar="LEVEL", type=str,
+        help=("Desired log output level (DEBUG, INFO, WARNING, ERROR). "
+              "Default: INFO."), default="INFO")
+    
+    args = parser.parse_args()
+    if not args.log_level in LOG_LEVELS:
+        raise ValueError("Illegal log-level argument: {}".format(args.log_level))
+    logging.root.setLevel(LOG_LEVELS[args.log_level])
+        
+    try:
+        if not os.path.isdir(args.backup_dir):
+            os.makedirs(args.backup_dir)
+            
+        if not args.password:
+            args.password = getpass.getpass("Enter password: ")
+        
+        with GarminClient(args.username, args.password) as client:
+            # already backed up activities (stored in backup-dir)
+            backed_up_activities = set(get_backed_up_ids(args.backup_dir))
+            log.info("{} contains {} backed up activities.".format(
+                args.backup_dir, len(backed_up_activities)))
+
+            # get all activity ids from Garmin account
+            log.info("retrieving activities for {} ...".format(args.username))
+            all_activities = set(client.list_activity_ids())
+            log.info("account has a total of {} activities.".format(
+                len(all_activities)))
+            
+            missing_activities = all_activities - backed_up_activities
+            log.info("activities that haven't been backed up: {}".format(
+                len(missing_activities)))
+            
+            for index, id in enumerate(missing_activities):
+                log.info("backing up activity {} ({} out of {}) ...".format(
+                    id, index+1, len(missing_activities)))
+                garminexport.util.save_activity(
+                    client, id, args.backup_dir)
+    except Exception as e:
+        exc_type, exc_value, exc_traceback = sys.exc_info()
+        log.error(u"failed with exception: %s", e)
+        raise
+