From bffd81a23103cd0a6780687e85b20d730f52b56e Mon Sep 17 00:00:00 2001 From: petergardfjall Date: Tue, 11 Nov 2014 13:45:11 +0100 Subject: [PATCH] refactoring --- README.md | 36 +++++++++--- garminexport.py | 47 +++------------ garminexport/garminclient.py | 5 +- garminexport/util.py | 53 +++++++++++++++++ incremental_backup.py | 108 +++++++++++++++++++++++++++++++++++ 5 files changed, 201 insertions(+), 48 deletions(-) create mode 100644 garminexport/util.py create mode 100755 incremental_backup.py diff --git a/README.md b/README.md index 47af00b..bcd7023 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,15 @@ garminexport ============ -The Garmin Connect activity exporter is a program that downloads all activities -for a given [Garmin Connect](http://connect.garmin.com/) account and stores them locally on the user's computer. +The Garmin Connect activity exporter is a program that downloads *all* +activities for a given [Garmin Connect](http://connect.garmin.com/) +account and stores them locally on the user's computer. + +The directory also contains an ``incremental_backup.py`` program that can be +used for incremental backups of your account. This script only downloads +activities that haven't already been downloaded to a certain backup directory. +It is typically a quicker alternative (except for the first time when all +activities will need to be downloaded). + Prerequisites ============= @@ -12,6 +20,7 @@ assumes that you have [Python 2.7](https://www.python.org/download/releases/2.7/ It also assumes that you have registered an account at [Garmin Connect](http://connect.garmin.com/). + Getting started =============== Create and activate a new virtual environment to create an isolated development @@ -24,16 +33,17 @@ Install the required dependencies in this virtual environment: pip install -r requirements.txt -Run the program -=============== -The program is run as follows (use the ``--help`` flag for a list of + +Running the export program +========================== +The export program is run as follows (use the ``--help`` flag for a list of available options). ./garminexport.py Once started, the program will prompt you for your account password and then -log in to your Garmin Connect account to download all activities to a destination -directory on your machine. +log in to your Garmin Connect account to download *all* activities to a +destination directory on your machine. For each activity, these files are stored: @@ -53,6 +63,18 @@ Each activity file is prefixed by its upload timestamp and its activity id. +Running the incremental backup program +====================================== +The incremental backup program is run in a similar fashion to the export +program (use the ``--help`` flag for a list of available options): + + ./incremental_backup.py --backup-dir=activities + +In this example, it will only download activities that aren't already in +the ``activities/`` directory. Note: The incremental backup program saves +the same files for each activity as the export program (see above). + + Library import ============== To install the development version of this library in your local Python diff --git a/garminexport.py b/garminexport.py index ed875cd..45d636f 100755 --- a/garminexport.py +++ b/garminexport.py @@ -3,15 +3,11 @@ and stores them locally on the user's computer. """ import argparse -import codecs -from datetime import datetime import getpass from garminexport.garminclient import GarminClient -import io -import json +import garminexport.util import logging import os -import shutil import sys import traceback @@ -53,7 +49,9 @@ if __name__ == "__main__": logging.root.setLevel(LOG_LEVELS[args.log_level]) try: - os.makedirs(args.destination) + if not os.path.isdir(args.destination): + os.makedirs(args.destination) + if not args.password: args.password = getpass.getpass("Enter password: ") @@ -61,39 +59,10 @@ if __name__ == "__main__": log.info("fetching activities for {} ...".format(args.username)) activity_ids = client.list_activity_ids() for index, id in enumerate(activity_ids): - log.info("processing activity {} out of {} ...".format( - index+1, len(activity_ids))) - activity_summary = client.get_activity_summary(id) - activity_details = client.get_activity_details(id) - activity_gpx = client.get_activity_gpx(id) - activity_tcx = client.get_activity_tcx(id) - activity_fit = client.get_activity_fit(id) - - # for each activitity save the summary, details and GPX file. - creation_millis = activity_summary["activity"]["uploadDate"]["millis"] - timestamp = datetime.fromtimestamp(int(creation_millis)/1000.0) - filename_prefix = "{}_{}".format( - timestamp.strftime("%Y%m%d-%H%M%S"), id) - path_prefix = os.path.join(args.destination, filename_prefix) - - summary_file = path_prefix + "_summary.json" - details_file = path_prefix + "_details.json" - gpx_file = path_prefix + ".gpx" - tcx_file = path_prefix + ".tcx" - fit_file = path_prefix + ".fit" - with codecs.open(summary_file, encoding="utf-8", mode="w") as f: - f.write(json.dumps( - activity_summary, ensure_ascii=False, indent=4)) - with codecs.open(details_file, encoding="utf-8", mode="w") as f: - f.write(json.dumps( - activity_details, ensure_ascii=False, indent=4)) - with codecs.open(gpx_file, encoding="utf-8", mode="w") as f: - f.write(activity_gpx) - with codecs.open(tcx_file, encoding="utf-8", mode="w") as f: - f.write(activity_tcx) - if activity_fit: - with open(fit_file, mode="wb") as f: - f.write(activity_fit) + log.info("processing activity {} ({} out of {}) ...".format( + id, index+1, len(activity_ids))) + garminexport.util.save_activity( + client, id, args.destination) except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() log.error(u"failed with exception: %s", e) diff --git a/garminexport/garminclient.py b/garminexport/garminclient.py index b253cb7..312f062 100755 --- a/garminexport/garminclient.py +++ b/garminexport/garminclient.py @@ -180,7 +180,8 @@ class GarminClient(object): :returns: A list of activity identifiers. :rtype: list of str """ - log.info("fetching activities {} through {} ...".format(start_index, start_index+max_limit-1)) + log.debug("fetching activities {} through {} ...".format( + start_index, start_index+max_limit-1)) response = self.session.get( "https://connect.garmin.com/proxy/activity-search-service-1.2/json/activities", params={"start": start_index, "limit": max_limit}) if response.status_code != 200: @@ -292,5 +293,5 @@ class GarminClient(object): # fit file returned from server is in a zip archive zipped_fit_file = response.content zip = zipfile.ZipFile(StringIO(zipped_fit_file), mode="r") - # return the ".fit" entry from the zip archive + # return the ".fit" entry from the zip archive return zip.open(str(activity_id) + ".fit").read() diff --git a/garminexport/util.py b/garminexport/util.py new file mode 100644 index 0000000..4d8dadc --- /dev/null +++ b/garminexport/util.py @@ -0,0 +1,53 @@ +#! /usr/bin/env python +"""A module with utility functions.""" + +import codecs +import json +from datetime import datetime +import os + +def save_activity(client, activity_id, destination): + """Downloads a certain Garmin Connect activity and saves it + to a given destination directory. + + :param client: A :class:`garminexport.garminclient.GarminClient` + instance that is assumed to be connected. + :type client: :class:`garminexport.garminclient.GarminClient` + :param activity_id: Activity identifier. + :type activity_id: int + :param destination: Destination directory (assumed to exist already). + :type destination: str + + """ + activity_summary = client.get_activity_summary(activity_id) + activity_details = client.get_activity_details(activity_id) + activity_gpx = client.get_activity_gpx(activity_id) + activity_tcx = client.get_activity_tcx(activity_id) + activity_fit = client.get_activity_fit(activity_id) + + # save activitity summary, details and GPX, TCX and FIT file. + creation_millis = activity_summary["activity"]["uploadDate"]["millis"] + timestamp = datetime.fromtimestamp(int(creation_millis)/1000.0) + filename_prefix = "{}_{}".format( + timestamp.strftime("%Y%m%d-%H%M%S"), activity_id) + path_prefix = os.path.join(destination, filename_prefix) + + summary_file = path_prefix + "_summary.json" + details_file = path_prefix + "_details.json" + gpx_file = path_prefix + ".gpx" + tcx_file = path_prefix + ".tcx" + fit_file = path_prefix + ".fit" + with codecs.open(summary_file, encoding="utf-8", mode="w") as f: + f.write(json.dumps( + activity_summary, ensure_ascii=False, indent=4)) + with codecs.open(details_file, encoding="utf-8", mode="w") as f: + f.write(json.dumps( + activity_details, ensure_ascii=False, indent=4)) + with codecs.open(gpx_file, encoding="utf-8", mode="w") as f: + f.write(activity_gpx) + with codecs.open(tcx_file, encoding="utf-8", mode="w") as f: + f.write(activity_tcx) + if activity_fit: + with open(fit_file, mode="wb") as f: + f.write(activity_fit) + diff --git a/incremental_backup.py b/incremental_backup.py new file mode 100755 index 0000000..e65acae --- /dev/null +++ b/incremental_backup.py @@ -0,0 +1,108 @@ +#! /usr/bin/env python +"""Performs (incremental) backups of activities for a given Garmin Connect +account. +The activities are stored in a local directory on the user's computer. +The backups are incremental, meaning that only activities that aren't already +stored in the backup directory will be downloaded. +""" +import argparse +import getpass +from garminexport.garminclient import GarminClient +import garminexport.util +import logging +import os +import re +import sys +import traceback + +logging.basicConfig( + level=logging.INFO, format="%(asctime)-15s [%(levelname)s] %(message)s") +log = logging.getLogger(__name__) + +LOG_LEVELS = { + "DEBUG": logging.DEBUG, + "INFO": logging.INFO, + "WARNING": logging.WARNING, + "ERROR": logging.ERROR +} +"""Command-line (string-based) log-level mapping to logging module levels.""" + +def get_backed_up_ids(backup_dir): + """Return all activitiy ids that have been backed up in the + given backup directory. + + :rtype: list of int + """ + # backed up activities follow this pattern + activity_file_pattern = r'[0-9]+\-[0-9]+_([0-9]+)_summary\.json' + + backed_up_ids = [] + dir_entries = os.listdir(backup_dir) + for entry in dir_entries: + activity_match = re.search(activity_file_pattern, entry) + if activity_match: + backed_up_id = int(activity_match.group(1)) + backed_up_ids.append(backed_up_id) + return backed_up_ids + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=( + "Performs incremental backups of activities for a " + "given Garmin Connect account. Only activities that " + "aren't already stored in the backup directory will " + "be downloaded.")) + # positional args + parser.add_argument( + "username", metavar="", type=str, help="Account user name.") + # optional args + parser.add_argument( + "--password", type=str, help="Account password.") + parser.add_argument( + "--backup-dir", metavar="DIR", type=str, + help=("Destination directory for downloaded activities. Default: " + "./activities/"), default=os.path.join(".", "activities")) + parser.add_argument( + "--log-level", metavar="LEVEL", type=str, + help=("Desired log output level (DEBUG, INFO, WARNING, ERROR). " + "Default: INFO."), default="INFO") + + args = parser.parse_args() + if not args.log_level in LOG_LEVELS: + raise ValueError("Illegal log-level argument: {}".format(args.log_level)) + logging.root.setLevel(LOG_LEVELS[args.log_level]) + + try: + if not os.path.isdir(args.backup_dir): + os.makedirs(args.backup_dir) + + if not args.password: + args.password = getpass.getpass("Enter password: ") + + with GarminClient(args.username, args.password) as client: + # already backed up activities (stored in backup-dir) + backed_up_activities = set(get_backed_up_ids(args.backup_dir)) + log.info("{} contains {} backed up activities.".format( + args.backup_dir, len(backed_up_activities))) + + # get all activity ids from Garmin account + log.info("retrieving activities for {} ...".format(args.username)) + all_activities = set(client.list_activity_ids()) + log.info("account has a total of {} activities.".format( + len(all_activities))) + + missing_activities = all_activities - backed_up_activities + log.info("activities that haven't been backed up: {}".format( + len(missing_activities))) + + for index, id in enumerate(missing_activities): + log.info("backing up activity {} ({} out of {}) ...".format( + id, index+1, len(missing_activities))) + garminexport.util.save_activity( + client, id, args.backup_dir) + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + log.error(u"failed with exception: %s", e) + raise +