diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index e6470a1b..00223e05 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -105,96 +105,8 @@ jobs: install_options: dist/*.whl - name: Test module --version works using the installed wheel - # If more than one module in src/ replace with module name to test - run: python -m $(ls src | head -1) --version - - container: - needs: [lint, dist, test] - runs-on: ubuntu-latest - - permissions: - contents: read - packages: write - - env: - TEST_TAG: "testing" - - steps: - - name: Checkout - uses: actions/checkout@v4 - - # image names must be all lower case - - name: Generate image repo name - run: echo IMAGE_REPOSITORY=ghcr.io/$(tr '[:upper:]' '[:lower:]' <<< "${{ github.repository }}") >> $GITHUB_ENV - - - name: Download wheel and lockfiles - uses: actions/download-artifact@v3 - with: - path: artifacts/ - - - name: Log in to GitHub Docker Registry - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and export to Docker local cache - uses: docker/build-push-action@v5 - with: - # Note build-args, context, file, and target must all match between this - # step and the later build-push-action, otherwise the second build-push-action - # will attempt to build the image again - build-args: | - PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl - context: artifacts/ - file: ./Dockerfile - target: runtime - load: true - tags: ${{ env.TEST_TAG }} - # If you have a long docker build (2+ minutes), uncomment the - # following to turn on caching. For short build times this - # makes it a little slower - #cache-from: type=gha - #cache-to: type=gha,mode=max - - - name: Test cli works in cached runtime image - run: docker run docker.io/library/${{ env.TEST_TAG }} --version - - - name: Create tags for publishing image - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.IMAGE_REPOSITORY }} - tags: | - type=ref,event=tag - type=raw,value=latest, enable=${{ github.ref_type == 'tag' }} - # type=edge,branch=main - # Add line above to generate image for every commit to given branch, - # and uncomment the end of if clause in next step - - - name: Push cached image to container registry - if: github.ref_type == 'tag' # || github.ref_name == 'main' - uses: docker/build-push-action@v5 - # This does not build the image again, it will find the image in the - # Docker cache and publish it - with: - # Note build-args, context, file, and target must all match between this - # step and the previous build-push-action, otherwise this step will - # attempt to build the image again - build-args: | - PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl - context: artifacts/ - file: ./Dockerfile - target: runtime - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + # Check that the command line entry point works + run: gphotos-sync --version release: # upload to PyPI and make a release on every tag @@ -229,3 +141,76 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_TOKEN }} + + make-container: + needs: [lint, dist, test] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: checkout + uses: actions/checkout@v2 + + - uses: actions/download-artifact@v2 + with: + name: dist + path: dist + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Log in to GitHub Docker Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ghcr.io/${{ github.repository }} + # github repo and dockerhub tag must match for this to work + ${{ github.repository }} + # all pull requests share a single tag 'pr' + tags: | + type=ref,event=branch + type=ref,event=tag + type=raw,value=latest,enable=${{ github.event_name != 'pull_request' }} + type=raw,value=pr + + # required for multi-arch build + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Build runtime image + uses: docker/build-push-action@v3 + with: + file: .devcontainer/Dockerfile + context: . + platforms: linux/amd64,linux/arm/v7,linux/arm64/v8 + push: true + build-args: BASE=python:3.10-slim + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/docs/reference/api.rst b/docs/reference/api.rst index c1e95363..7392b413 100644 --- a/docs/reference/api.rst +++ b/docs/reference/api.rst @@ -13,10 +13,10 @@ This is the internal code reference for gphotos_sync Version number as calculated by setuptools_scm -.. automodule:: gphotos_sync.Main +.. automodule:: gphotos_sync.__main__ :members: - ``gphotos_sync.Main`` + ``gphotos_sync.__main__`` ----------------------------------------- .. automodule:: gphotos_sync.BaseMedia :members: diff --git a/src/gphotos_sync/Main.py b/src/gphotos_sync/Main.py deleted file mode 100644 index 170cb528..00000000 --- a/src/gphotos_sync/Main.py +++ /dev/null @@ -1,523 +0,0 @@ -# coding: utf8 -import logging -import os -import sys -from argparse import ArgumentParser, Namespace -from datetime import datetime -from pathlib import Path -from typing import Optional -from xmlrpc.client import DateTime - -from appdirs import AppDirs - -from gphotos_sync import Utils, __version__ -from gphotos_sync.authorize import Authorize -from gphotos_sync.Checks import do_check, get_check -from gphotos_sync.GoogleAlbumsSync import GoogleAlbumsSync -from gphotos_sync.GooglePhotosDownload import GooglePhotosDownload # type: ignore -from gphotos_sync.GooglePhotosIndex import GooglePhotosIndex -from gphotos_sync.LocalData import LocalData -from gphotos_sync.LocalFilesScan import LocalFilesScan -from gphotos_sync.Logging import setup_logging -from gphotos_sync.restclient import RestClient -from gphotos_sync.Settings import Settings - -if os.name == "nt": - import subprocess - - orig_Popen = subprocess.Popen - - class Popen_patch(subprocess.Popen): - def __init__(self, *args, **kargs): - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - kargs["startupinfo"] = startupinfo - super().__init__(*args, **kargs) - - subprocess.Popen = Popen_patch # type: ignore -else: - import fcntl - -APP_NAME = "gphotos-sync" -log = logging.getLogger(__name__) - - -class GooglePhotosSyncMain: - def __init__(self): - self.data_store: LocalData - self.google_photos_client: RestClient - self.google_photos_idx: GooglePhotosIndex - self.google_photos_down: GooglePhotosDownload - self.google_albums_sync: GoogleAlbumsSync - self.local_files_scan: LocalFilesScan - self._start_date: Optional[DateTime] - self._end_date = Optional[DateTime] - - self.auth: Authorize - - try: - version_string = "version: {}, database schema version {}".format( - __version__, LocalData.VERSION - ) - except TypeError: - version_string = "(version not available)" - - parser = ArgumentParser( - epilog=version_string, description="Google Photos download tool" - ) - parser.add_argument( - "--version", - action="store_true", - help="report version and exit", - ) - parser.add_argument( - "root_folder", - help="root of the local folders to download into", - nargs="?", - ) - album_group = parser.add_mutually_exclusive_group() - album_group.add_argument( - "--album", - action="store", - help="only synchronize the contents of a single album. " - 'use quotes e.g. "album name" for album names with spaces', - ) - album_group.add_argument( - "--album-regex", - action="store", - metavar="REGEX", - help="""only synchronize albums that match regular expression. - regex is case insensitive and unanchored. e.g. to select two albums: - "^(a full album name|another full name)$" """, - ) - parser.add_argument( - "--log-level", - help="Set log level. Options: critical, error, warning, info, debug, trace. " - "trace logs all Google API calls to a file with suffix .trace", - default="warning", - ) - parser.add_argument( - "--logfile", - action="store", - help="full path to debug level logfile, default: /gphotos.log. " - "If a directory is specified then a unique filename will be " - "generated.", - ) - parser.add_argument( - "--compare-folder", - action="store", - help="DEPRECATED: root of the local folders to compare to the Photos Library", - ) - parser.add_argument( - "--favourites-only", - action="store_true", - help="only download media marked as favourite (star)", - ) - parser.add_argument( - "--flush-index", - action="store_true", - help="delete the index db, re-scan everything", - ) - parser.add_argument( - "--rescan", - action="store_true", - help="rescan entire library, ignoring last scan date. Use this if you " - "have added photos to the library that " - "predate the last sync, or you have deleted some of the local " - "files", - ) - parser.add_argument( - "--retry-download", - action="store_true", - help="check for the existence of files marked as already downloaded " - "and re-download any missing ones. Use " - "this if you have deleted some local files", - ) - parser.add_argument( - "--skip-video", action="store_true", help="skip video types in sync" - ) - parser.add_argument( - "--skip-shared-albums", - action="store_true", - help="skip albums that only appear in 'Sharing'", - ) - parser.add_argument( - "--album-date-by-first-photo", - action="store_true", - help="Make the album date the same as its earliest " - "photo. The default is its last photo", - ) - parser.add_argument( - "--start-date", - help="Set the earliest date of files to sync" "format YYYY-MM-DD", - default=None, - ) - parser.add_argument( - "--end-date", - help="Set the latest date of files to sync" "format YYYY-MM-DD", - default=None, - ) - parser.add_argument( - "--db-path", - help="Specify a pre-existing folder for the index database. " - "Defaults to the root of the local download folders", - default=None, - ) - parser.add_argument( - "--albums-path", - help="Specify a folder for the albums " - "Defaults to the 'albums' in the local download folders", - default="albums", - ) - parser.add_argument( - "--photos-path", - help="Specify a folder for the photo files. " - "Defaults to the 'photos' in the local download folders", - default="photos", - ) - parser.add_argument( - "--use-flat-path", - action="store_true", - help="Mandate use of a flat directory structure ('YYYY-MMM') and not " - "a nested one ('YYYY/MM') . ", - ) - parser.add_argument( - "--omit-album-date", - action="store_true", - help="Don't include year and month in album folder names.", - ) - parser.add_argument( - "--album-invert", - action="store_true", - help="Inverts the sorting direction of files within an album. " - "Default sorting is descending from newest to olders. " - "This causes it to be the other way around.", - ) - parser.add_argument("--new-token", action="store_true", help="Request new token") - parser.add_argument( - "--index-only", - action="store_true", - help="Only build the index of files in .gphotos.db - no downloads", - ) - parser.add_argument( - "--skip-index", - action="store_true", - help="Use index from previous run and start download immediately", - ) - parser.add_argument( - "--do-delete", - action="store_true", - help="""Remove local copies of files that were deleted. - Must be used with --flush-index since the deleted items must be removed - from the index""", - ) - parser.add_argument( - "--skip-files", - action="store_true", - help="Dont download files, just refresh the album links (for testing)", - ) - parser.add_argument( - "--skip-albums", action="store_true", help="Dont download albums (for testing)" - ) - parser.add_argument( - "--use-hardlinks", - action="store_true", - help="Use hardlinks instead of symbolic links in albums and comparison" - " folders", - ) - parser.add_argument( - "--no-album-index", - action="store_true", - help="only index the photos library - skip indexing of folder contents " - "(for testing)", - ) - parser.add_argument( - "--case-insensitive-fs", - action="store_true", - help="add this flag if your filesystem is case insensitive", - ) - parser.add_argument( - "--max-retries", - help="Set the number of retries on network timeout / failures", - type=int, - default=20, - ) - parser.add_argument( - "--max-threads", - help="Set the number of concurrent threads to use for parallel " - "download of media - reduce this number if network load is " - "excessive", - type=int, - default=20, - ) - parser.add_argument( - "--secret", - help="Path to client secret file (by default this is in the " - "application config directory)", - ) - parser.add_argument( - "--archived", - action="store_true", - help="Download media items that have been marked as archived", - ) - parser.add_argument( - "--progress", - action="store_true", - help="show progress of indexing and downloading in warning log", - ) - parser.add_argument( - "--max-filename", - help="Set the maxiumum filename length for target filesystem." - "This overrides the automatic detection.", - default=0, - ) - parser.add_argument( - "--ntfs", - action="store_true", - help="Declare that the target filesystem is ntfs (or ntfs like)." - "This overrides the automatic detection.", - ) - parser.add_argument( - "--month-format", - action="store", - metavar="FMT", - help="Configure the month/day formatting for the album folder/file " - "path (default: %%m%%d).", - default="%m%d", - ) - parser.add_argument( - "--path-format", - action="store", - metavar="FMT", - help="Configure the formatting for the album folder/file path. The " - "formatting can include up to 2 positionals arguments; `month` and " - "`album_name`. The default value is `{0} {1}`." - "When used with --use-flat-path option, it can include up to 3 " - "positionals arguments; `year`, `month` and `album_name`. In this case " - "the default value is `{0}-{1} {2}`", - default=None, - ) - parser.add_argument( - "--port", - help="Set the port for login flow redirect", - type=int, - default=8080, - ) - parser.add_argument( - "--image-timeout", - help="Set the time in seconds to wait for an image to download", - type=int, - default=60, - ) - parser.add_argument( - "--video-timeout", - help="Set the time in seconds to wait for a video to download", - type=int, - default=2000, - ) - parser.add_help = True - - def setup(self, args: Namespace, db_path: Path): - root_folder = Path(args.root_folder).absolute() - - compare_folder = None - if args.compare_folder: - compare_folder = Path(args.compare_folder).absolute() - app_dirs = AppDirs(APP_NAME) - - self.data_store = LocalData(db_path, args.flush_index) - - credentials_file = db_path / ".gphotos.token" - if args.secret: - secret_file = Path(args.secret) - else: - secret_file = Path(app_dirs.user_config_dir) / "client_secret.json" - if args.new_token and credentials_file.exists(): - credentials_file.unlink() - - scope = [ - "https://www.googleapis.com/auth/photoslibrary.readonly", - "https://www.googleapis.com/auth/photoslibrary.sharing", - ] - photos_api_url = ( - "https://photoslibrary.googleapis.com/$discovery" "/rest?version=v1" - ) - - self.auth = Authorize( - scope, - credentials_file, - secret_file, - int(args.max_retries), - port=args.port, - ) - self.auth.authorize() - - settings = Settings( - start_date=Utils.string_to_date(args.start_date), # type: ignore - end_date=Utils.string_to_date(args.end_date), # type: ignore - shared_albums=not args.skip_shared_albums, - album_index=not args.no_album_index, - use_start_date=args.album_date_by_first_photo, - album=args.album, - album_regex=args.album_regex, - favourites_only=args.favourites_only, - retry_download=args.retry_download, - case_insensitive_fs=args.case_insensitive_fs, - include_video=not args.skip_video, - rescan=args.rescan, - archived=args.archived, - photos_path=Path(args.photos_path), - albums_path=Path(args.albums_path), - use_flat_path=args.use_flat_path, - max_retries=int(args.max_retries), - max_threads=int(args.max_threads), - omit_album_date=args.omit_album_date, - album_invert=args.album_invert, - use_hardlinks=args.use_hardlinks, - progress=args.progress, - ntfs_override=args.ntfs, - month_format=args.month_format, - path_format=args.path_format, - image_timeout=args.image_timeout, - video_timeout=args.video_timeout, - ) - - self.google_photos_client = RestClient( - photos_api_url, self.auth.session # type: ignore - ) - self.google_photos_idx = GooglePhotosIndex( - self.google_photos_client, root_folder, self.data_store, settings - ) - self.google_photos_down = GooglePhotosDownload( - self.google_photos_client, root_folder, self.data_store, settings - ) - self.google_albums_sync = GoogleAlbumsSync( - self.google_photos_client, - root_folder, - self.data_store, - args.flush_index or args.retry_download or args.rescan, - settings, - ) - if args.compare_folder: - self.local_files_scan = LocalFilesScan( - root_folder, compare_folder, self.data_store # type: ignore - ) - - def do_sync(self, args: Namespace): - files_downloaded = 0 - with self.data_store: - if not args.skip_index: - if not args.skip_files and not args.album and not args.album_regex: - self.google_photos_idx.index_photos_media() - - if not args.index_only: - if not args.skip_files: - files_downloaded = self.google_photos_down.download_photo_media() - - if ( - not args.skip_albums - and not args.skip_index - and (files_downloaded > 0 or args.skip_files or args.rescan) - ) or (args.album is not None or args.album_regex is not None): - self.google_albums_sync.index_album_media() - # run download again to pick up files indexed in albums only - if not args.index_only: - if not args.skip_files: - files_downloaded = ( - self.google_photos_down.download_photo_media() - ) - - if not args.index_only: - if ( - not args.skip_albums - and (files_downloaded > 0 or args.skip_files or args.rescan) - or (args.album is not None or args.album_regex is not None) - ): - self.google_albums_sync.create_album_content_links() - if args.do_delete: - self.google_photos_idx.check_for_removed() - - if args.compare_folder: - if not args.skip_index: - self.local_files_scan.scan_local_files() - self.google_photos_idx.get_extra_meta() - self.local_files_scan.find_missing_gphotos() - - def start(self, args: Namespace): - self.do_sync(args) - - @staticmethod - def fs_checks(root_folder: Path, args): - Utils.minimum_date(root_folder) - # store the root folder filesystem checks globally for all to inspect - do_check(root_folder, int(args.max_filename), bool(args.ntfs)) - - # check if symlinks are supported - # NTFS supports symlinks, but is_symlink() fails - if not args.ntfs: - if not get_check().is_symlink: # type: ignore - args.skip_albums = True - - # check if file system is case sensitive - if not args.case_insensitive_fs: - if not get_check().is_case_sensitive: # type: ignore - args.case_insensitive_fs = True - - return args - - def main(self, test_args: Optional[dict] = None): - start_time = datetime.now() - args = self.parser.parse_args(test_args) # type: ignore - - if args.version: - print(__version__) - exit(0) - else: - if args.root_folder is None: - self.parser.print_help() - print("\nERROR: Please supply root_folder in which to save photos") - exit(1) - - root_folder = Path(args.root_folder).absolute() - db_path = Path(args.db_path) if args.db_path else root_folder - if not root_folder.exists(): - root_folder.mkdir(parents=True, mode=0o700) - - setup_logging(args.log_level, args.logfile, root_folder) - log.warning(f"gphotos-sync {__version__} {start_time}") - - args = self.fs_checks(root_folder, args) - - lock_file = db_path / "gphotos.lock" - fp = lock_file.open("w") - with fp: - try: - if os.name != "nt": - fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - log.warning("EXITING: database is locked") - sys.exit(0) - - log.info(self.version_string) - - # configure and launch - # noinspection PyBroadException - try: - self.setup(args, db_path) - self.start(args) - except KeyboardInterrupt: - log.error("User cancelled download") - log.debug("Traceback", exc_info=True) - exit(1) - except BaseException: - log.error("\nProcess failed.", exc_info=True) - exit(1) - finally: - log.warning("Done.") - - elapsed_time = datetime.now() - start_time - log.info("Elapsed time = %s", elapsed_time) - - -def main(): - GooglePhotosSyncMain().main() diff --git a/src/gphotos_sync/__main__.py b/src/gphotos_sync/__main__.py deleted file mode 100644 index 235986b2..00000000 --- a/src/gphotos_sync/__main__.py +++ /dev/null @@ -1,16 +0,0 @@ -from argparse import ArgumentParser - -from . import __version__ - -__all__ = ["main"] - - -def main(args=None): - parser = ArgumentParser() - parser.add_argument("--version", action="version", version=__version__) - args = parser.parse_args(args) - - -# test with: python -m gphotos_sync -if __name__ == "__main__": - main() diff --git a/tests/test_credentials/.gphotos.token b/tests/test_credentials/.gphotos.token index 7cf84e76..35c0ad9e 100644 --- a/tests/test_credentials/.gphotos.token +++ b/tests/test_credentials/.gphotos.token @@ -1 +1 @@ -{"access_token": "ya29.a0AfB_byAAnSEyVYmlxFIXG7A7lH8jkrSrd4RSnunI4rYyHfzdG-ZjSQiFxo5-OHV5rBgEnHoFMKK1Rj5PKxLCXVCEr-0yMiJs61wCCfzg36-UWQegZTV3kBotQH_Qk7HTkZcphp11fW5lbc3POClog-aOCfJLIbOe2UfA63MaCgYKAZsSARMSFQHGX2Mi3joogOa2oHnOerg_ojwUQg0174", "expires_in": 3599, "scope": ["https://www.googleapis.com/auth/photoslibrary.sharing", "https://www.googleapis.com/auth/photoslibrary.readonly"], "token_type": "Bearer", "expires_at": 1703971275.0828457, "refresh_token": "1//03CEqAzsnP-8PCgYIARAAGAMSNwF-L9Irz4_ilhRw0HIwVImT4gTCUPlV8YaCTYQiIjD4juWOI5eQh_-Rzh9nTmBND0jliOnabq4"} \ No newline at end of file +{"access_token": "ya29.a0AfB_byAreGK4x3LmWMoEdyxynFhxmVdluIku4_wrxknJNMivmNheLMhz1UT2bya7Oq_sKzgljfBXgEvVl5ODSq6XRu53doa0zOzeKJiEbl9KbdqqTlbVy2r5HV_FJ2weHCzihyhDkUtcFduPSsLQNFffLjZubPLvTGDtdN0aCgYKAXISARMSFQHGX2MiqhOMZleHPcJE2ubZkU7jvw0174", "expires_in": 3599, "scope": ["https://www.googleapis.com/auth/photoslibrary.sharing", "https://www.googleapis.com/auth/photoslibrary.readonly"], "token_type": "Bearer", "expires_at": 1704573379.582039, "refresh_token": "1//03CEqAzsnP-8PCgYIARAAGAMSNwF-L9Irz4_ilhRw0HIwVImT4gTCUPlV8YaCTYQiIjD4juWOI5eQh_-Rzh9nTmBND0jliOnabq4"} \ No newline at end of file diff --git a/tests/test_setup.py b/tests/test_setup.py index 085b1101..9c00b30e 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -4,9 +4,9 @@ from appdirs import AppDirs -from gphotos_sync import Main +from gphotos_sync import __main__ +from gphotos_sync.__main__ import GooglePhotosSyncMain from gphotos_sync.Checks import do_check -from gphotos_sync.Main import GooglePhotosSyncMain # if we are debugging requests library is too noisy logging.getLogger("requests").setLevel(logging.WARNING) @@ -23,8 +23,8 @@ class SetupDbAndCredentials: def __init__(self): # set up the test account credentials - Main.APP_NAME = "gphotos-sync-test" - app_dirs = AppDirs(Main.APP_NAME) + __main__.APP_NAME = "gphotos-sync-test" + app_dirs = AppDirs(__main__.APP_NAME) self.test_folder = Path(__file__).absolute().parent / "test_credentials" user_data = Path(app_dirs.user_data_dir) if not user_data.exists():