diff --git a/FAQ.md b/FAQ.md index 4537cb0f..65e11fec 100644 --- a/FAQ.md +++ b/FAQ.md @@ -211,32 +211,46 @@ database state back to the selected backend. ### Is there support for Multi-user setup? -There is a minimal support for multi-user setup via `state:sync` command. However, it still requires that you add your -backends as usual for single user setup and to use `state:sync` command, it's required that all backends have admin -access to be able to retrieve access-tokens for users. That means for Plex you need an admin token, and for -jellyfin/emby you need API key, not `user:password` limited access. +There are minimal support for multi-user setup via `state:sync` command. There are some requirements to get it working +correctly. The tools will try to match the users based on the name, and fallback on the `mapper.yaml` file if it's +provided. The tool will try to sync the users data between the backends. -To get started using `state:sync` command, as mentioned before setup your backends as normal, then create a -`/config/config/mapper.yaml` file if your backends doesn't have the same user. for example +#### Things that will get synced + +* Play status, i.e. watched/unwatched. +* Watch progress. + +#### Requirements to get the command working + +* All backends need to have admin level access, this is needed to inquiry about the users and generate the required + access tokens. +* That means for plex, it needs the admin token, to find it + check [plex article about it](https://support.plex.tv/articles/204059436-finding-an-authentication-token-x-plex-token/). +* For jellyfin/emby you need to use the API key, not the user password. You can generate api keys via Dashboard > + Advanced > API Keys. + +#### Whats the schema for the `mapper.yaml` file? + +The schema is simple, it's a list of users in the following format: ```yaml -- backend_name1: +- my_plex_server: name: "mike_jones" options: { } - backend_name2: + my_jellyfin_server: name: "jones_mike" options: { } - backend_name3: + my_emby_server: name: "mikeJones" options: { } -- backend_name1: +- my_emby_server: name: "jiji_jones" options: { } - backend_name2: + my_plex_server: name: "jones_jiji" options: { } - backend_name3: + my_jellyfin_server: name: "jijiJones" options: { } ``` @@ -244,20 +258,10 @@ To get started using `state:sync` command, as mentioned before setup your backen This yaml file helps map your users accounts in the different backends, so the tool can sync the correct user data. Then simply run `state:sync -v` it will generate the required tokens and match users data between the backends. -then sync the difference, Keep in mind that it will be slow and that's expected as it needs to do the same thing without -caching for all users servers and backends. it's recommended to not run this command frequently. as it's puts a lot of -load on the backends. By default, it will sync once every 3 hours. you can ofc change it to suit your needs. +then sync the difference. By default, the task is scheduled to run every 3 hour, you can change the schedule by +altering the `WS_CRON_SYNC_AT` environment variable via `ENV` page or `system:env` command. -> [!NOTE] -> Known issues: - -* Currently, `state:sync` doesn't have a way of syncing plex users that has PIN enabled. -* Majority of the command flags aren't working or not implemented yet. - -> [!IMPORTANT] -> Please keep in mind the new command is still in alpha stage, so things will probably break. Please report any bugs -> you encounter. Also, please make sure to have a backup of your data before running the command. just in-case, -> while we did test it on our live data, it's always better to be safe than sorry. +To have the task run automatically, you need to enable the task via the `WebUI > Tasks` page or `system:env` command. ---- diff --git a/NEWS.md b/NEWS.md index bb58bce8..9a5692ba 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,21 @@ # Old Updates +### 2024-12-30 + +We have removed the old environment variables `WS_CRON_PROGRESS` and `WS_CRON_PUSH` in favor of the new ones +`WS_SYNC_PROGRESS` and `WS_PUSH_ENABLED`. please update your environment variables accordingly. We have also added +new FAQ entry about watch progress syncing via [this link](FAQ.md#sync-watch-progress). + +### 2024-10-07 + +We have added a WebUI page for Custom GUIDs and stabilized on `v1.0` for the `guid.yaml` file spec. We strongly +recommend +to use the `WebUI` to manage the GUIDs, as it's much easier to use than editing the `guid.yaml` file directly. and both +the +`WebUI` and `API` have safeguards to prevent you from breaking the parser. For more information please check out the +associated +FAQ entry about it at [this link](FAQ.md#advanced-how-to-extend-the-guid-parser-to-support-more-guids-or-custom-ones). + ### 2024-09-14 We have recently added support for extending WatchState with more GUIDs, as of now, the support for it is done via @@ -14,46 +30,61 @@ or request the maintainer to add support for it. ### 2024-08-19 We have migrated the `state:push` task into the new events system, as such the old task `state:push` is now gone. -To enable the new event handler for push events, use the new environment variable `WS_PUSH_ENABLED` and set it to `true`. +To enable the new event handler for push events, use the new environment variable `WS_PUSH_ENABLED` and set it to +`true`. Right now, it's disabled by default. However, for people who had the old task enabled, it will reuse that setting. -Keep in mind, the new event handler is more efficient and will only push data when there is a change in the play state. And it's much faster +Keep in mind, the new event handler is more efficient and will only push data when there is a change in the play state. +And it's much faster than the old task. This event handler will push data within a minute of the change. -PS: Please enable the task by setting its new environment variable `WS_PUSH_ENABLED` to `true`. The old `WS_CRON_PUSH` is now gone. +PS: Please enable the task by setting its new environment variable `WS_PUSH_ENABLED` to `true`. The old `WS_CRON_PUSH` +is now gone. and will be removed in the future releases. ### 2024-08-18 -We have started migrating the old events system to a new one, so far we have migrated the `progress` and `requests` to it. As such, -The old tasks `state:progress` and `state:requests` are now gone. To control if you want to enable the watch progress, there is new -environment variable `WS_SYNC_PROGRESS` which you can set to `true` to enable the watch progress. It's disabled by default. +We have started migrating the old events system to a new one, so far we have migrated the `progress` and `requests` to +it. As such, +The old tasks `state:progress` and `state:requests` are now gone. To control if you want to enable the watch progress, +there is new +environment variable `WS_SYNC_PROGRESS` which you can set to `true` to enable the watch progress. It's disabled by +default. We will continue to migrate the rest of the events to the new system, and we will keep you updated. ### 2024-08-10 I have recently added new experimental feature, to play your content directly from the WebUI. This feature is still in -alpha, and missing a lot of features. But it's a start. Right now it does auto transcode on the fly to play any content in the browser. +alpha, and missing a lot of features. But it's a start. Right now it does auto transcode on the fly to play any content +in the browser. -The feature requires that you mount your media directories to the `WatchState` container similar to the `File integrity` feature. I have plans to expand -the feature to support more controls, however, right now it's only support basic subtitles streams and default audio stream or first audio stream. +The feature requires that you mount your media directories to the `WatchState` container similar to the `File integrity` +feature. I have plans to expand +the feature to support more controls, however, right now it's only support basic subtitles streams and default audio +stream or first audio stream. -The transcoder works by converting the media on the fly to `HLS` segments, and the subtitles are selectable via the player ui which are also converted to `vtt` format. +The transcoder works by converting the media on the fly to `HLS` segments, and the subtitles are selectable via the +player ui which are also converted to `vtt` format. -Expects bugs and issues, as the feature is still in alpha. But I would love to hear your feedback. You can play the media by visiting -the history page of the item you will see red play button on top right corner of the page. If the items has a play button, then you correctly mounted +Expects bugs and issues, as the feature is still in alpha. But I would love to hear your feedback. You can play the +media by visiting +the history page of the item you will see red play button on top right corner of the page. If the items has a play +button, then you correctly mounted the media directories. otherwise, the button be disabled with tooltip of `Media is inaccessible`. -The feature is not meant to replace your backend media player, the purpose of this feature is to quickly check the media without leaving the WebUI. +The feature is not meant to replace your backend media player, the purpose of this feature is to quickly check the media +without leaving the WebUI. ### 2024-08-01 -We recently enabled listening on tls connections via `8443` which can be controlled by `HTTPS_PORT` environment variable. +We recently enabled listening on tls connections via `8443` which can be controlled by `HTTPS_PORT` environment +variable. Before today, we simply only exposed the port via the `Dockerfile`, but we weren't listening for connections on it. However, please keep in mind that the certificate is self-signed, and you might get a warning from your browser. You can -either accept the warning or add the certificate to your trusted certificates. We strongly recommend using a reverse proxy. +either accept the warning or add the certificate to your trusted certificates. We strongly recommend using a reverse +proxy. instead of relying on self-signed certificates. ### 2024-07-22 @@ -62,14 +93,18 @@ We have recently added a new WebUI feature, `File integrity`, this feature will are reporting files that are not available on the disk. This feature is still in alpha, and we are working on improving it. -This feature `REQUIRES` that you mount your media directories to the `WatchState` container preferably as readonly. There is plans to add +This feature `REQUIRES` that you mount your media directories to the `WatchState` container preferably as readonly. +There is plans to add a path replacement feature to allow you change the pathing, but it's not implemented yet. -This feature will work on both local and remote cloud storages provided they are mounted into the container. We also may recommend not to -use this feature depending on how your cloud storage provider treats file stat calls. As it might lead to unnecessary money spending. and of course +This feature will work on both local and remote cloud storages provided they are mounted into the container. We also may +recommend not to +use this feature depending on how your cloud storage provider treats file stat calls. As it might lead to unnecessary +money spending. and of course it will be slower. -For more information about how we cache the stat calls, please refer to the [FAQ](FAQ.md#How-does-the-file-integrity-feature-works). +For more information about how we cache the stat calls, please refer to +the [FAQ](FAQ.md#How-does-the-file-integrity-feature-works). ### 2024-07-06 @@ -111,8 +146,10 @@ can be used. This environment variable can be enabled by setting `WS_API_AUTO=tr ### 2024-05-14 -We are happy to announce the beta testing of the `WebUI`. To get started on using it you just need to visit the url `http://localhost:8080` We are supposed to -enabled it by default tomorrow, but we decided to give you a head start. We are looking forward to your feedback. If you don't use the `WebUI` then you need to +We are happy to announce the beta testing of the `WebUI`. To get started on using it you just need to visit the url +`http://localhost:8080` We are supposed to +enabled it by default tomorrow, but we decided to give you a head start. We are looking forward to your feedback. If you +don't use the `WebUI` then you need to add the environment variable `WEBUI_ENABLED=0` in your `compose.yaml` file. and restart the container. ### 2024-05-13 @@ -128,8 +165,10 @@ Note: `WS_WEBUI_ENABLED` will be gone in few weeks, However it will still work f ### 2024-05-05 **Edit** - We received requests that people are exposing watchstate externally, and there was concern that having open -webhook endpoints might lead to abuse. As such, we have added a new environment variable `WS_SECURE_API_ENDPOINTS`. Simply set -the environment variable to `1` to secure the webhook endpoint. This means you have to add `?apikey=yourapikey` to the end +webhook endpoints might lead to abuse. As such, we have added a new environment variable `WS_SECURE_API_ENDPOINTS`. +Simply set +the environment variable to `1` to secure the webhook endpoint. This means you have to add `?apikey=yourapikey` to the +end of the webhook endpoint. ----- @@ -166,9 +205,12 @@ All commands that was accepting backend name as argument now accepts `-s, --sele the command interface more consistent and easier to use. Another breaking change is the removal of the `-c, --config` flag from all commands that was accepting it. This flag was -used to override the default `servers.yaml` file. This was not working as expected as there are more than just the `servers.yaml` -to consider like, the state of cache, and the state of the database. As such, we have removed this flag. However, we have -added a new environment variable called `WS_BACKENDS_FILE` which can be used to override the default `servers.yaml` file. +used to override the default `servers.yaml` file. This was not working as expected as there are more than just the +`servers.yaml` +to consider like, the state of cache, and the state of the database. As such, we have removed this flag. However, we +have +added a new environment variable called `WS_BACKENDS_FILE` which can be used to override the default `servers.yaml` +file. We strongly recommend not to use it as it might lead to unexpected behavior. We started working on a `Web API` which hopefully will lead to a `web frontend` to manage the tool. This is a long diff --git a/README.md b/README.md index cdcf720d..9ec03414 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,13 @@ out of the box, this tool support `Jellyfin`, `Plex` and `Emby` media servers. ## Updates +### 2025-01-24 + +We are excited to share that multi-user sync is now fully supported! Our first goal was to make sure the feature worked, +and since releasing it, we’ve worked hard to improve it based on feedback and testing. We’re now confident that it works +as expected and are happy to invite you to start using it. To learn more and get started, please check out the FAQ entry +here: [this link](FAQ.md#is-there-support-for-multi-user-setup). + ### 2025-01-18 Due to popular demand, we finally have added the ability to sync all users data, however, it's limited to only @@ -22,22 +29,6 @@ API key for jellyfin and emby. Enable the task and let it run, it will sync all Please read the FAQ entry about it at [this link](FAQ.md#is-there-support-for-multi-user-setup). -### 2024-12-30 - -We have removed the old environment variables `WS_CRON_PROGRESS` and `WS_CRON_PUSH` in favor of the new ones -`WS_SYNC_PROGRESS` and `WS_PUSH_ENABLED`. please update your environment variables accordingly. We have also added -new FAQ entry about watch progress syncing via [this link](FAQ.md#sync-watch-progress). - -### 2024-10-07 - -We have added a WebUI page for Custom GUIDs and stabilized on `v1.0` for the `guid.yaml` file spec. We strongly -recommend -to use the `WebUI` to manage the GUIDs, as it's much easier to use than editing the `guid.yaml` file directly. and both -the -`WebUI` and `API` have safeguards to prevent you from breaking the parser. For more information please check out the -associated -FAQ entry about it at [this link](FAQ.md#advanced-how-to-extend-the-guid-parser-to-support-more-guids-or-custom-ones). - --- Refer to [NEWS](NEWS.md) for old updates. diff --git a/config/config.php b/config/config.php index f671e2fe..d6494f65 100644 --- a/config/config.php +++ b/config/config.php @@ -278,7 +278,7 @@ SyncCommand::TASK_NAME => [ 'command' => SyncCommand::ROUTE, 'name' => SyncCommand::TASK_NAME, - 'info' => '[Alpha stage] Sync All users play state. Read the FAQ.', + 'info' => 'Sync ALL users play state. Read the FAQ.', 'enabled' => (bool)env('WS_CRON_SYNC', false), 'timer' => $checkTaskTimer((string)env('WS_CRON_SYNC_AT', '9 */3 * * *'), '9 */3 * * *'), 'args' => env('WS_CRON_SYNC_ARGS', '-v'), diff --git a/src/Backends/Plex/Action/GetUserToken.php b/src/Backends/Plex/Action/GetUserToken.php index c4dd9d2e..a46e0a02 100644 --- a/src/Backends/Plex/Action/GetUserToken.php +++ b/src/Backends/Plex/Action/GetUserToken.php @@ -67,7 +67,7 @@ private function getUserToken(Context $context, int|string $userId, string $user $pin = ag($context->options, Options::PLEX_USER_PIN); - $this->logger->debug('Requesting temporary access token for [{backend}] user [{username}]{pin}', [ + $this->logger->debug("Requesting temporary access token for '{backend}' user '{username}'{pin}", [ 'backend' => $context->backendName, 'username' => $username, 'user_id' => $userId, diff --git a/src/Commands/State/SyncCommand.php b/src/Commands/State/SyncCommand.php index 2b2a91cb..0a3c58a8 100644 --- a/src/Commands/State/SyncCommand.php +++ b/src/Commands/State/SyncCommand.php @@ -6,6 +6,7 @@ use App\Backends\Common\Cache as BackendCache; use App\Backends\Common\ClientInterface as iClient; +use App\Backends\Plex\PlexClient; use App\Command; use App\Libs\Attributes\DI\Inject; use App\Libs\Attributes\Route\Cli; @@ -91,6 +92,7 @@ protected function configure(): void InputOption::VALUE_NONE, 'Mapper option. Always update the locally stored metadata from backend.' ) + ->addOption('regenerate-tokens', 'g', InputOption::VALUE_NONE, 'Generate new tokens for all users.') ->addOption('include-main-user', null, InputOption::VALUE_NONE, 'Include main user in sync.') ->setHelp( r( @@ -292,12 +294,9 @@ protected function process(iInput $input, iOutput $output): int unset($backend); - $this->logger->notice( - "SYSTEM: Getting users list from '{backends}'.", - [ - 'backends' => join(', ', array_map(fn($backend) => $backend['name'], $backends)) - ] - ); + $this->logger->notice("SYSTEM: Getting users list from '{backends}'.", [ + 'backends' => join(', ', array_map(fn($backend) => $backend['name'], $backends)) + ]); $users = []; @@ -310,10 +309,9 @@ protected function process(iInput $input, iOutput $output): int ]); try { - foreach ($client->getUsersList(['tokens' => true]) as $user) { + foreach ($client->getUsersList() as $user) { /** @var array $info */ $info = $backend; - $info['token'] = ag($user, 'token', ag($backend, 'token')); $info['user'] = ag($user, 'id', ag($info, 'user')); $info['backendName'] = r("{backend}_{user}", [ 'backend' => ag($backend, 'name'), @@ -324,8 +322,12 @@ protected function process(iInput $input, iOutput $output): int $info = ag_delete($info, 'options.' . Options::ADMIN_TOKEN); $info = ag_set($info, 'options.' . Options::ALT_NAME, ag($backend, 'name')); $info = ag_set($info, 'options.' . Options::ALT_ID, ag($backend, 'user')); + if (PlexClient::CLIENT_NAME === ucfirst(ag($backend, 'type'))) { + $info = ag_set($info, 'token', 'reuse_or_generate_token'); + $info = ag_set($info, 'options.' . Options::PLEX_USER_NAME, ag($user, 'name')); + $info = ag_set($info, 'options.' . Options::PLEX_USER_UUID, ag($user, 'uuid')); + } - unset($info['class']); $user['backend'] = ag($backend, 'name'); $user['client_data'] = $info; $users[] = $user; @@ -387,26 +389,63 @@ protected function process(iInput $input, iOutput $output): int $list = []; $displayName = null; - $configFile = ConfigFile::open(r(fixPath(Config::get('path') . '/users/{user}/servers.yaml'), [ + $perUser = ConfigFile::open(r(fixPath(Config::get('path') . '/users/{user}/servers.yaml'), [ 'user' => $userName ]), 'yaml', autoSave: true, autoCreate: true); - $configFile->setLogger($this->logger); + $perUser->setLogger($this->logger); + + $regenerateTokens = $input->getOption('regenerate-tokens'); foreach (ag($user, 'backends', []) as $backend) { $name = ag($backend, 'client_data.backendName'); $clientData = ag($backend, 'client_data'); $clientData['name'] = $name; - if (false === $configFile->has($name)) { + if (false === $perUser->has($name)) { $data = $clientData; $data = ag_set($data, 'import.lastSync', null); $data = ag_set($data, 'export.lastSync', null); $data = ag_delete($data, ['webhook', 'name', 'backendName', 'displayName']); - $configFile->set($name, $data); + $perUser->set($name, $data); } else { - $clientData = ag_delete($clientData, 'import.lastSync'); - $clientData = ag_delete($clientData, 'export.lastSync'); - $clientData = array_replace_recursive($configFile->get($name), $clientData); + $clientData = ag_delete($clientData, ['token', 'import.lastSync', 'export.lastSync']); + $clientData = array_replace_recursive($perUser->get($name), $clientData); + } + + try { + if (true === $regenerateTokens || 'reuse_or_generate_token' === ag($clientData, 'token')) { + /** @var iClient $client */ + $client = ag($backend, 'client_data.class'); + assert($client instanceof iClient); + if (PlexClient::CLIENT_NAME === $client->getType()) { + $clientData['token'] = $client->getUserToken( + ag($clientData, 'options.' . Options::PLEX_USER_UUID), + ag($clientData, 'options.' . Options::PLEX_USER_NAME) + ); + $perUser->set("{$name}.token", $clientData['token']); + } + } + } catch (Throwable $e) { + $this->logger->error( + "Failed to generate access token for '{user}: {name}' backend. '{error}' at '{file}:{line}'.", + [ + 'name' => $name, + 'user' => $userName, + 'error' => [ + 'kind' => $e::class, + 'line' => $e->getLine(), + 'message' => $e->getMessage(), + 'file' => after($e->getFile(), ROOT_PATH), + ], + 'exception' => [ + 'file' => $e->getFile(), + 'line' => $e->getLine(), + 'kind' => get_class($e), + 'message' => $e->getMessage(), + ], + ] + ); + continue; } $clientData['class'] = makeBackend($clientData, $name, [ @@ -417,8 +456,8 @@ protected function process(iInput $input, iOutput $output): int $displayName = ag($backend, 'client_data.displayName', '??'); if (false === $input->getOption('dry-run')) { - $configFile->set("{$name}.import.lastSync", time()); - $configFile->set("{$name}.export.lastSync", time()); + $perUser->set("{$name}.import.lastSync", time()); + $perUser->set("{$name}.export.lastSync", time()); } } @@ -430,7 +469,7 @@ protected function process(iInput $input, iOutput $output): int ]); assert($perUserMapper instanceof iEImport); - $this->handleImport($perUserMapper, $displayName, $list, $input->getOption('force-full'), $configFile); + $this->handleImport($perUserMapper, $displayName, $list, $input->getOption('force-full'), $perUser); assert($perUserMapper instanceof MemoryMapper); /** @var MemoryMapper $changes */ @@ -458,7 +497,7 @@ protected function process(iInput $input, iOutput $output): int } } - $this->handleExport($displayName); + $this->handleExport($displayName, ag($user, 'backends', [])); $end = makeDate(); $this->logger->notice("SYSTEM: Completed syncing user '{name}' -> '{list}' in '{time.duration}'s", [ @@ -485,6 +524,7 @@ protected function process(iInput $input, iOutput $output): int $this->logger->info("SYSTEM: Memory usage after reset '{memory}'.", [ 'memory' => getMemoryUsage(), ]); + $perUser->persist(); } return self::SUCCESS; @@ -564,11 +604,14 @@ protected function handleImport( Message::add('response.size', 0); } - protected function handleExport(string $name): void + protected function handleExport(string $name, array $backends): void { $total = count($this->queue->getQueue()); if ($total < 1) { - $this->logger->notice("SYSTEM: No play state changes detected for '{name}' backends.", ['name' => $name]); + $this->logger->notice("SYSTEM: No play state changes detected for '{name}: {backends}'.", [ + 'name' => $name, + 'backends' => join(', ', array_keys($backends)) + ]); return; } @@ -796,7 +839,9 @@ private function generate_users_list(array $users, bool $includeMainUser, array } // Ensure $matchedUser['client_data']['options'] is an array - if (!isset($matchedUser['client_data']['options']) || !is_array($matchedUser['client_data']['options'])) { + if (!isset($matchedUser['client_data']['options']) || !is_array( + $matchedUser['client_data']['options'] + )) { $matchedUser['client_data']['options'] = []; } diff --git a/src/Libs/Options.php b/src/Libs/Options.php index e6f31ce0..53988102 100644 --- a/src/Libs/Options.php +++ b/src/Libs/Options.php @@ -27,6 +27,7 @@ final class Options public const string DUMP_PAYLOAD = 'DUMP_PAYLOAD'; public const string ADMIN_TOKEN = 'ADMIN_TOKEN'; public const string PLEX_USER_UUID = 'plex_user_uuid'; + public const string PLEX_USER_NAME = 'plex_user_name'; public const string NO_THROW = 'NO_THROW'; public const string NO_LOGGING = 'NO_LOGGING'; public const string MAX_EPISODE_RANGE = 'MAX_EPISODE_RANGE';