Refactor main script logic into an incremental_backup library function
The overall intent is to make it easier for third-party clients to implement incremental backups as simple function calls to the garminexport library, rather than executing the garminbackup.py script. Co-authored-by: Stanislav Khrapov <stanislav.khrapov@dbschenker.com>
This commit is contained in:
Stanislav Khrapov
GitHub
parent
847cb84a94
commit
8cd27fcb19
121
garminbackup.py
121
garminbackup.py
@ -1,124 +1,29 @@
|
||||
#! /usr/bin/env python
|
||||
"""Performs (incremental) backups of activities for a given Garmin Connect
|
||||
account.
|
||||
"""This python script calls garminexport.garminbackup module with CLI parsed arguments
|
||||
and performs (incremental) backups of activities for a given Garmin Connect account.
|
||||
The activities are stored in a local directory on the user's computer.
|
||||
The backups are incremental, meaning that only activities that aren't already
|
||||
stored in the backup directory will be downloaded.
|
||||
"""
|
||||
import argparse
|
||||
from datetime import timedelta
|
||||
import getpass
|
||||
from garminexport.garminclient import GarminClient
|
||||
import garminexport.backup
|
||||
from garminexport.backup import export_formats
|
||||
from garminexport.retryer import (
|
||||
Retryer, ExponentialBackoffDelayStrategy, MaxRetriesStopStrategy)
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)-15s [%(levelname)s] %(message)s")
|
||||
from garminexport.cli import parse_args
|
||||
from garminexport.incremental_backup import incremental_backup
|
||||
from garminexport.logging_config import LOG_LEVELS
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)-15s [%(levelname)s] %(message)s")
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
LOG_LEVELS = {
|
||||
"DEBUG": logging.DEBUG,
|
||||
"INFO": logging.INFO,
|
||||
"WARNING": logging.WARNING,
|
||||
"ERROR": logging.ERROR
|
||||
}
|
||||
"""Command-line (string-based) log-level mapping to logging module levels."""
|
||||
|
||||
DEFAULT_MAX_RETRIES = 7
|
||||
"""The default maximum number of retries to make when fetching a single activity."""
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Performs incremental backups of activities for a "
|
||||
"given Garmin Connect account. Only activities that "
|
||||
"aren't already stored in the backup directory will "
|
||||
"be downloaded."))
|
||||
# positional args
|
||||
parser.add_argument(
|
||||
"username", metavar="<username>", type=str, help="Account user name.")
|
||||
# optional args
|
||||
parser.add_argument(
|
||||
"--password", type=str, help="Account password.")
|
||||
parser.add_argument(
|
||||
"--backup-dir", metavar="DIR", type=str,
|
||||
help=("Destination directory for downloaded activities. Default: "
|
||||
"./activities/"), default=os.path.join(".", "activities"))
|
||||
parser.add_argument(
|
||||
"--log-level", metavar="LEVEL", type=str,
|
||||
help=("Desired log output level (DEBUG, INFO, WARNING, ERROR). "
|
||||
"Default: INFO."), default="INFO")
|
||||
parser.add_argument(
|
||||
"-f", "--format", choices=export_formats,
|
||||
default=None, action='append',
|
||||
help=("Desired output formats ("+', '.join(export_formats)+"). "
|
||||
"Default: ALL."))
|
||||
parser.add_argument(
|
||||
"-E", "--ignore-errors", action='store_true',
|
||||
help="Ignore errors and keep going. Default: FALSE")
|
||||
parser.add_argument(
|
||||
"--max-retries", metavar="NUM", default=DEFAULT_MAX_RETRIES,
|
||||
type=int, help="The maximum number of retries to make on failed attempts to fetch an activity. Exponential backoff will be used, meaning that the delay between successive attempts will double with every retry, starting at one second. DEFAULT: %d" % DEFAULT_MAX_RETRIES)
|
||||
|
||||
args = parser.parse_args()
|
||||
if not args.log_level in LOG_LEVELS:
|
||||
raise ValueError("Illegal log-level: {}".format(args.log_level))
|
||||
|
||||
# if no --format was specified, all formats are to be backed up
|
||||
args.format = args.format if args.format else export_formats
|
||||
log.info("backing up formats: %s", ", ".join(args.format))
|
||||
|
||||
args = parse_args()
|
||||
logging.root.setLevel(LOG_LEVELS[args.log_level])
|
||||
|
||||
try:
|
||||
if not os.path.isdir(args.backup_dir):
|
||||
os.makedirs(args.backup_dir)
|
||||
incremental_backup(username=args.username,
|
||||
password=args.password,
|
||||
backup_dir=args.backup_dir,
|
||||
format=args.format,
|
||||
max_retries=args.max_retries)
|
||||
|
||||
if not args.password:
|
||||
args.password = getpass.getpass("Enter password: ")
|
||||
|
||||
# set up a retryer that will handle retries of failed activity
|
||||
# downloads
|
||||
retryer = Retryer(
|
||||
delay_strategy=ExponentialBackoffDelayStrategy(
|
||||
initial_delay=timedelta(seconds=1)),
|
||||
stop_strategy=MaxRetriesStopStrategy(args.max_retries))
|
||||
|
||||
|
||||
with GarminClient(args.username, args.password) as client:
|
||||
# get all activity ids and timestamps from Garmin account
|
||||
log.info("scanning activities for %s ...", args.username)
|
||||
activities = set(retryer.call(client.list_activities))
|
||||
log.info("account has a total of %d activities", len(activities))
|
||||
|
||||
missing_activities = garminexport.backup.need_backup(
|
||||
activities, args.backup_dir, args.format)
|
||||
backed_up = activities - missing_activities
|
||||
log.info("%s contains %d backed up activities",
|
||||
args.backup_dir, len(backed_up))
|
||||
|
||||
log.info("activities that aren't backed up: %d",
|
||||
len(missing_activities))
|
||||
|
||||
for index, activity in enumerate(missing_activities):
|
||||
id, start = activity
|
||||
log.info("backing up activity %d from %s (%d out of %d) ..." % (id, start, index+1, len(missing_activities)))
|
||||
try:
|
||||
garminexport.backup.download(
|
||||
client, activity, retryer, args.backup_dir,
|
||||
args.format)
|
||||
except Exception as e:
|
||||
log.error(u"failed with exception: %s", e)
|
||||
if not args.ignore_errors:
|
||||
raise
|
||||
except Exception as e:
|
||||
exc_type, exc_value, exc_traceback = sys.exc_info()
|
||||
log.error(u"failed with exception: %s", str(e))
|
||||
|
||||
@ -2,10 +2,9 @@
|
||||
"""
|
||||
import codecs
|
||||
import json
|
||||
from datetime import datetime
|
||||
import dateutil.parser
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
49
garminexport/cli.py
Normal file
49
garminexport/cli.py
Normal file
@ -0,0 +1,49 @@
|
||||
import argparse
|
||||
import os
|
||||
|
||||
from garminexport.backup import export_formats
|
||||
|
||||
DEFAULT_MAX_RETRIES = 7
|
||||
"""The default maximum number of retries to make when fetching a single activity."""
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""Parse CLI arguments.
|
||||
|
||||
:return: Namespace object holding parsed arguments as attributes.
|
||||
This object may be directly used by garminexport/garminbackup.py.
|
||||
"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Performs incremental backups of activities for a "
|
||||
"given Garmin Connect account. Only activities that "
|
||||
"aren't already stored in the backup directory will "
|
||||
"be downloaded."))
|
||||
# positional args
|
||||
parser.add_argument(
|
||||
"username", metavar="<username>", type=str, help="Account user name.")
|
||||
# optional args
|
||||
parser.add_argument(
|
||||
"--password", type=str, help="Account password.")
|
||||
parser.add_argument(
|
||||
"--backup-dir", metavar="DIR", type=str,
|
||||
help="Destination directory for downloaded activities. Default: ./activities/",
|
||||
default=os.path.join(".", "activities"))
|
||||
parser.add_argument(
|
||||
"--log-level", metavar="LEVEL", type=str,
|
||||
help="Desired log output level (DEBUG, INFO, WARNING, ERROR). Default: INFO.",
|
||||
default="INFO")
|
||||
parser.add_argument(
|
||||
"-f", "--format", choices=export_formats,
|
||||
default=None, action='append',
|
||||
help="Desired output formats (" + ', '.join(export_formats) + "). Default: ALL.")
|
||||
parser.add_argument(
|
||||
"-E", "--ignore-errors", action='store_true',
|
||||
help="Ignore errors and keep going. Default: FALSE")
|
||||
parser.add_argument(
|
||||
"--max-retries", metavar="NUM", default=DEFAULT_MAX_RETRIES,
|
||||
type=int, help=("The maximum number of retries to make on failed attempts to fetch an activity. "
|
||||
"Exponential backoff will be used, meaning that the delay between successive attempts "
|
||||
"will double with every retry, starting at one second. DEFAULT: %d") % DEFAULT_MAX_RETRIES)
|
||||
|
||||
return parser.parse_args()
|
||||
72
garminexport/incremental_backup.py
Normal file
72
garminexport/incremental_backup.py
Normal file
@ -0,0 +1,72 @@
|
||||
#! /usr/bin/env python
|
||||
import getpass
|
||||
import logging
|
||||
import os
|
||||
from datetime import timedelta
|
||||
|
||||
import garminexport.backup
|
||||
from garminexport.backup import export_formats
|
||||
from garminexport.garminclient import GarminClient
|
||||
from garminexport.retryer import Retryer, ExponentialBackoffDelayStrategy, MaxRetriesStopStrategy
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def incremental_backup(username: str,
|
||||
password: str = None,
|
||||
backup_dir: str = os.path.join(".", "activities"),
|
||||
format: str = 'ALL',
|
||||
ignore_errors: bool = False,
|
||||
max_retries: int = 7):
|
||||
"""Performs (incremental) backups of activities for a given Garmin Connect account.
|
||||
|
||||
:param username: Garmin Connect user name
|
||||
:param password: Garmin Connect user password. Default: None. If not provided, would be asked interactively.
|
||||
:param backup_dir: Destination directory for downloaded activities. Default: ./activities/".
|
||||
:param format: Desired output formats (json_summary, json_details, gpx, tcx, fit). Default: ALL.
|
||||
:param ignore_errors: Ignore errors and keep going. Default: False.
|
||||
:param max_retries: The maximum number of retries to make on failed attempts to fetch an activity.
|
||||
Exponential backoff will be used, meaning that the delay between successive attempts
|
||||
will double with every retry, starting at one second. Default: 7.
|
||||
|
||||
The activities are stored in a local directory on the user's computer.
|
||||
The backups are incremental, meaning that only activities that aren't already
|
||||
stored in the backup directory will be downloaded.
|
||||
"""
|
||||
# if no --format was specified, all formats are to be backed up
|
||||
format = format if format else export_formats
|
||||
log.info("backing up formats: %s", ", ".join(format))
|
||||
|
||||
if not os.path.isdir(backup_dir):
|
||||
os.makedirs(backup_dir)
|
||||
|
||||
if not password:
|
||||
password = getpass.getpass("Enter password: ")
|
||||
|
||||
# set up a retryer that will handle retries of failed activity downloads
|
||||
retryer = Retryer(
|
||||
delay_strategy=ExponentialBackoffDelayStrategy(initial_delay=timedelta(seconds=1)),
|
||||
stop_strategy=MaxRetriesStopStrategy(max_retries))
|
||||
|
||||
with GarminClient(username, password) as client:
|
||||
# get all activity ids and timestamps from Garmin account
|
||||
log.info("scanning activities for %s ...", username)
|
||||
activities = set(retryer.call(client.list_activities))
|
||||
log.info("account has a total of %d activities", len(activities))
|
||||
|
||||
missing_activities = garminexport.backup.need_backup(activities, backup_dir, format)
|
||||
backed_up = activities - missing_activities
|
||||
log.info("%s contains %d backed up activities", backup_dir, len(backed_up))
|
||||
|
||||
log.info("activities that aren't backed up: %d", len(missing_activities))
|
||||
|
||||
for index, activity in enumerate(missing_activities):
|
||||
id, start = activity
|
||||
log.info("backing up activity %d from %s (%d out of %d) ..." % (
|
||||
id, start, index + 1, len(missing_activities)))
|
||||
try:
|
||||
garminexport.backup.download(client, activity, retryer, backup_dir, format)
|
||||
except Exception as e:
|
||||
log.error(u"failed with exception: %s", e)
|
||||
if not ignore_errors:
|
||||
raise
|
||||
9
garminexport/logging_config.py
Normal file
9
garminexport/logging_config.py
Normal file
@ -0,0 +1,9 @@
|
||||
import logging
|
||||
|
||||
LOG_LEVELS = {
|
||||
"DEBUG": logging.DEBUG,
|
||||
"INFO": logging.INFO,
|
||||
"WARNING": logging.WARNING,
|
||||
"ERROR": logging.ERROR
|
||||
}
|
||||
"""Command-line (string-based) log-level mapping to logging module levels."""
|
||||
@ -3,30 +3,21 @@
|
||||
Connect account and stores it locally on the user's computer.
|
||||
"""
|
||||
import argparse
|
||||
from datetime import timedelta
|
||||
import getpass
|
||||
from garminexport.garminclient import GarminClient
|
||||
import garminexport.backup
|
||||
from garminexport.retryer import (
|
||||
Retryer, ExponentialBackoffDelayStrategy, MaxRetriesStopStrategy)
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from datetime import timedelta
|
||||
|
||||
import dateutil.parser
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)-15s [%(levelname)s] %(message)s")
|
||||
log = logging.getLogger(__name__)
|
||||
import garminexport.backup
|
||||
from garminexport.garminclient import GarminClient
|
||||
from garminexport.logging_config import LOG_LEVELS
|
||||
from garminexport.retryer import Retryer, ExponentialBackoffDelayStrategy, MaxRetriesStopStrategy
|
||||
|
||||
LOG_LEVELS = {
|
||||
"DEBUG": logging.DEBUG,
|
||||
"INFO": logging.INFO,
|
||||
"WARNING": logging.WARNING,
|
||||
"ERROR": logging.ERROR
|
||||
}
|
||||
"""Command-line (string-based) log-level mapping to logging module levels."""
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)-15s [%(levelname)s] %(message)s")
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
|
||||
@ -4,22 +4,15 @@ Connect account.
|
||||
"""
|
||||
import argparse
|
||||
import getpass
|
||||
from garminexport.garminclient import GarminClient
|
||||
import logging
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)-15s [%(levelname)s] %(message)s")
|
||||
from garminexport.garminclient import GarminClient
|
||||
from garminexport.logging_config import LOG_LEVELS
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)-15s [%(levelname)s] %(message)s")
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
LOG_LEVELS = {
|
||||
"DEBUG": logging.DEBUG,
|
||||
"INFO": logging.INFO,
|
||||
"WARNING": logging.WARNING,
|
||||
"ERROR": logging.ERROR
|
||||
}
|
||||
"""Command-line (string-based) log-level mapping to logging module levels."""
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user