Line data Source code
1 : /*
2 : * SPDX-License-Identifier: GPL-3.0-only
3 : * MuseScore-Studio-CLA-applies
4 : *
5 : * MuseScore Studio
6 : * Music Composition & Notation
7 : *
8 : * Copyright (C) 2026 MuseScore Limited and others
9 : *
10 : * This program is free software: you can redistribute it and/or modify
11 : * it under the terms of the GNU General Public License version 3 as
12 : * published by the Free Software Foundation.
13 : *
14 : * This program is distributed in the hope that it will be useful,
15 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 : * GNU General Public License for more details.
18 : *
19 : * You should have received a copy of the GNU General Public License
20 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
21 : */
22 : #include "convertfiletoscoreservice.h"
23 :
24 : #include <algorithm>
25 : #include <array>
26 : #include <chrono>
27 : #include <memory>
28 : #include <optional>
29 : #include <unordered_map>
30 :
31 : #include <QUrl>
32 :
33 : #include "project/types/filecategory.h"
34 : #include "project/projecterrors.h"
35 :
36 : #include "network/networkerrors.h"
37 : #include "cloud/clouderrors.h"
38 : #include "multiwindows/resourcelockguard.h"
39 :
40 : #include "global/serialization/json.h"
41 : #include "global/types/bytearray.h"
42 : #include "global/io/path.h"
43 : #include "global/io/ioretcodes.h"
44 : #include "global/log.h"
45 :
46 : using namespace mu::project;
47 : using namespace muse;
48 : using namespace muse::cloud;
49 :
50 21 : static bool isRetryableError(const Ret& ret)
51 : {
52 21 : switch (static_cast<muse::network::Err>(ret.code())) {
53 0 : case muse::network::Err::Timeout:
54 : case muse::network::Err::NetworkError:
55 0 : return true;
56 21 : default: break;
57 : }
58 :
59 21 : switch (static_cast<muse::cloud::Err>(ret.code())) {
60 20 : case muse::cloud::Err::NetworkError:
61 : case muse::cloud::Err::Status429_RateLimitExceeded:
62 : case muse::cloud::Err::Status500_InternalServerError:
63 : case muse::cloud::Err::UnknownStatusCode:
64 20 : return true;
65 1 : default: break;
66 : }
67 :
68 1 : return false;
69 : }
70 :
71 22 : static FileCategory resolveFileCategory(const io::path_t& path, const ConvertConfig& config)
72 : {
73 22 : const QString ext = QString::fromStdString(muse::io::suffix(path));
74 22 : if (ext == "pdf") {
75 7 : return FileCategory::Pdf;
76 : }
77 :
78 15 : if (config.omr.images.allowedExtensions.contains(ext)) {
79 8 : return FileCategory::Image;
80 : }
81 :
82 7 : if (config.audio2score.file.allowedExtensions.contains(ext)) {
83 5 : return FileCategory::Audio;
84 : }
85 :
86 2 : return FileCategory::Unknown;
87 22 : }
88 :
89 3 : static std::string errorCodeToString(ConvertErrorCode code)
90 : {
91 3 : switch (code) {
92 0 : case ConvertErrorCode::Unknown: return "Something went wrong";
93 0 : case ConvertErrorCode::UnsupportedFormat: return "This file format is not supported";
94 9 : case ConvertErrorCode::FileTooLarge: return "The file is too large";
95 0 : case ConvertErrorCode::TooManyFiles: return "Too many files were provided";
96 0 : case ConvertErrorCode::FileOrLinkRequired: return "A file or a link is required";
97 0 : case ConvertErrorCode::InvalidLink: return "The provided link is invalid";
98 0 : case ConvertErrorCode::RateLimited: return "Too many conversion requests, please try again later";
99 0 : case ConvertErrorCode::MsczNotReady: return "The score is not ready yet";
100 0 : case ConvertErrorCode::NoNeedReview: return "This score does not require a review";
101 0 : case ConvertErrorCode::ReviewRequired: return "A review must be submitted first";
102 0 : case ConvertErrorCode::CommentRequired: return "A comment is required";
103 0 : case ConvertErrorCode::VariousFileIssues: return "Invalid format or file type";
104 0 : case ConvertErrorCode::TooComplex: return "The file is too large or there is a problem with access to this file";
105 0 : case ConvertErrorCode::DontRecognizeNotes: return "Invalid file, could not recognize notes";
106 0 : case ConvertErrorCode::GeneralFailure: return "Something went wrong";
107 0 : case ConvertErrorCode::BadParams: return "Invalid conversion parameters";
108 : }
109 0 : return std::string();
110 : }
111 :
112 : static const std::string WATCHED_CONVERTS_RESOURCE_NAME("WATCHED_CONVERTS");
113 :
114 72 : static std::string convertIdAndType(ConvertType type, int itemId)
115 : {
116 144 : return "id: " + std::to_string(itemId) + ", type: " + convertTypeToString(type);
117 : }
118 :
119 23 : void ConvertFileToScoreService::init()
120 : {
121 24 : TRACEFUNC;
122 :
123 23 : m_timer.setInterval(MIN_RETRY_INTERVAL_MS); // poll once a minute
124 :
125 23 : QObject::connect(&m_timer, &QTimer::timeout, [this]() { poll(); });
126 :
127 : //! NOTE: fallback is used if fetchConfig() fails
128 23 : m_config.omr.pdf.maxFileSizeBytes = 78643200;
129 23 : m_config.omr.pdf.maxFiles = 1;
130 23 : m_config.omr.pdf.maxPages = 50;
131 161 : m_config.omr.images.allowedExtensions = { "jpeg", "jpg", "png" };
132 23 : m_config.omr.images.maxFileSizeBytes = 78643200;
133 23 : m_config.omr.images.maxFiles = 15;
134 161 : m_config.audio2score.file.allowedExtensions = { "mp3", "wav", "flac" };
135 23 : m_config.audio2score.file.maxFileSizeBytes = 52428800;
136 23 : m_config.audio2score.file.maxFiles = 1;
137 23 : m_config.audio2score.link.maxLength = 2048;
138 23 : m_config.audio2score.link.allowedSources = LinkSource::YouTube | LinkSource::AudioCom;
139 :
140 : //! NOTE: prefetch and cache convert config
141 23 : museScoreComService()->convert()->fetchConfig().onResolve(this, [this](const RetVal<ConvertConfig>& config) {
142 22 : if (!config.ret) {
143 1 : LOGE() << "Could not prefetch convert config: " << config.ret.toString();
144 : } else {
145 21 : m_config = config.val;
146 : }
147 22 : });
148 :
149 23 : multiwindowsProvider()->resourceChanged().onReceive(this, [this](const std::string& resourceName) {
150 0 : if (resourceName == WATCHED_CONVERTS_RESOURCE_NAME && !m_isSaving) {
151 0 : loadWatchedScores();
152 :
153 0 : if (!m_watchedScores.empty() && !m_timer.isActive()) {
154 0 : m_timer.start();
155 : }
156 :
157 0 : m_watchedScoresChanged.notify();
158 : }
159 0 : });
160 23 : }
161 :
162 4 : void ConvertFileToScoreService::resumeConvert()
163 : {
164 4 : loadWatchedScores();
165 :
166 4 : if (m_watchedScores.empty()) {
167 0 : return;
168 : }
169 :
170 4 : LOGI() << "Resuming watching " << m_watchedScores.size() << " pending conversion(s)";
171 :
172 4 : m_watchedScoresChanged.notify();
173 4 : m_timer.start();
174 4 : poll();
175 : }
176 :
177 3 : const ConvertConfig& ConvertFileToScoreService::config() const
178 : {
179 3 : return m_config;
180 : }
181 :
182 2 : bool ConvertFileToScoreService::isFileSupported(const io::path_t& path) const
183 : {
184 2 : return resolveFileCategory(path, m_config) != FileCategory::Unknown;
185 : }
186 :
187 13 : RetVal<ConvertFilesValidation> ConvertFileToScoreService::validateFiles(const io::paths_t& paths) const
188 : {
189 13 : if (paths.empty()) {
190 1 : return RetVal<ConvertFilesValidation>::make_ret(make_ret(Err::ConvertValidationFailed));
191 : }
192 :
193 12 : std::optional<FileCategory> firstCategory;
194 12 : qint64 totalSizeBytes = 0;
195 :
196 29 : for (const io::path_t& path : paths) {
197 20 : const FileCategory category = resolveFileCategory(path, m_config);
198 20 : if (category == FileCategory::Unknown) {
199 1 : return RetVal<ConvertFilesValidation>::make_ret(make_ret(Err::ConvertUnsupportedFormat));
200 : }
201 :
202 19 : if (!firstCategory) {
203 11 : firstCategory = category;
204 8 : } else if (category != firstCategory) {
205 1 : return RetVal<ConvertFilesValidation>::make_ret(make_ret(Err::ConvertMixedFileTypes));
206 : }
207 :
208 18 : const RetVal<uint64_t> fileSizeResult = fileSystem()->fileSize(path);
209 18 : if (!fileSizeResult.ret) {
210 0 : return RetVal<ConvertFilesValidation>::make_ret(fileSizeResult.ret);
211 : }
212 :
213 18 : const qint64 fileSizeBytes = static_cast<qint64>(fileSizeResult.val);
214 :
215 18 : if (category == FileCategory::Audio
216 4 : && m_config.audio2score.file.maxFileSizeBytes > 0 && fileSizeBytes > m_config.audio2score.file.maxFileSizeBytes) {
217 1 : return RetVal<ConvertFilesValidation>::make_ret(make_ret(Err::ConvertAudioFileTooLarge));
218 : }
219 :
220 17 : totalSizeBytes += fileSizeBytes;
221 18 : }
222 :
223 9 : if (firstCategory == FileCategory::Pdf
224 9 : && m_config.omr.pdf.maxFiles > 0 && int(paths.size()) > m_config.omr.pdf.maxFiles) {
225 1 : return RetVal<ConvertFilesValidation>::make_ret(make_ret(Err::ConvertMultiplePdfFiles));
226 : }
227 :
228 8 : if (firstCategory == FileCategory::Audio) {
229 2 : if (m_config.audio2score.file.maxFiles > 0 && int(paths.size()) > m_config.audio2score.file.maxFiles) {
230 1 : return RetVal<ConvertFilesValidation>::make_ret(make_ret(Err::ConvertTooManyAudioFiles));
231 : }
232 :
233 2 : return RetVal<ConvertFilesValidation>::make_ok(ConvertFilesValidation { ConvertType::Audio2Score, FileCategory::Audio });
234 : }
235 :
236 6 : if (firstCategory == FileCategory::Image
237 6 : && m_config.omr.images.maxFiles > 0 && int(paths.size()) > m_config.omr.images.maxFiles) {
238 1 : return RetVal<ConvertFilesValidation>::make_ret(make_ret(Err::ConvertTooManyImages));
239 : }
240 :
241 : //! NOTE: maxFileSizeBytes is a combined budget across all selected images
242 : //! (or the single file's own size, for a PDF)
243 5 : const qint64 maxFileSizeBytes = firstCategory == FileCategory::Image
244 5 : ? m_config.omr.images.maxFileSizeBytes
245 5 : : m_config.omr.pdf.maxFileSizeBytes;
246 5 : if (maxFileSizeBytes > 0 && totalSizeBytes > maxFileSizeBytes) {
247 2 : if (firstCategory == FileCategory::Image) {
248 1 : return RetVal<ConvertFilesValidation>::make_ret(make_ret(Err::ConvertCombinedImageTooLarge));
249 : }
250 1 : return RetVal<ConvertFilesValidation>::make_ret(make_ret(Err::ConvertFileTooLarge));
251 : }
252 :
253 3 : return RetVal<ConvertFilesValidation>::make_ok(ConvertFilesValidation { ConvertType::Omr, *firstCategory });
254 : }
255 :
256 7 : Ret ConvertFileToScoreService::validateLink(const QUrl& link) const
257 : {
258 7 : if (!link.isValid()) {
259 0 : return make_ret(Err::ConvertUnsupportedLink, link.errorString().toStdString());
260 : }
261 :
262 7 : const int maxLength = m_config.audio2score.link.maxLength;
263 7 : if (maxLength > 0 && link.toString().length() > maxLength) {
264 1 : return make_ret(Err::ConvertUnsupportedLink);
265 : }
266 :
267 6 : const QString host = link.host().toLower();
268 :
269 12 : if (m_config.audio2score.link.allowedSources.testFlag(LinkSource::YouTube)
270 6 : && (host == "youtube.com" || host.endsWith(".youtube.com") || host == "youtu.be")) {
271 2 : return make_ok();
272 : }
273 :
274 8 : if (m_config.audio2score.link.allowedSources.testFlag(LinkSource::AudioCom)
275 4 : && (host == "audio.com" || host.endsWith(".audio.com"))) {
276 1 : return make_ok();
277 : }
278 :
279 3 : return make_ret(Err::ConvertUnsupportedLink);
280 6 : }
281 :
282 54 : RetVal<Progress> ConvertFileToScoreService::uploadToConvert(const ConvertInput& input, const muse::String& convertedScoreName)
283 : {
284 54 : const io::paths_t paths = convertPathsOf(input);
285 54 : const QUrl link = convertLinkOf(input);
286 :
287 54 : IF_ASSERT_FAILED(!paths.empty() || link.isValid()) {
288 0 : return RetVal<Progress>::make_ret(make_ret(Err::ConvertValidationFailed));
289 : }
290 :
291 54 : IF_ASSERT_FAILED(io::isAllowedFileName(io::path_t(convertedScoreName))) {
292 0 : return RetVal<Progress>::make_ret(make_ret(Err::ConvertValidationFailed));
293 : }
294 :
295 54 : ConvertFileDataList files;
296 54 : files.reserve(paths.size());
297 107 : for (const io::path_t& path : paths) {
298 54 : RetVal<ByteArray> fileData = fileSystem()->readFile(path);
299 54 : if (!fileData.ret) {
300 1 : return RetVal<Progress>::make_ret(fileData.ret);
301 : }
302 :
303 53 : files.push_back(ConvertFileData { std::move(fileData.val), io::filename(path) });
304 54 : }
305 :
306 53 : const ConvertType type = convertTypeOf(input);
307 106 : auto data = std::make_shared<const ConvertUploadData>(ConvertUploadData { type, std::move(files), link,
308 53 : convertedScoreName.toQString() });
309 53 : ProgressPtr uploadProgress = museScoreComService()->convert()->uploadToConvert(data);
310 :
311 53 : LOGI() << "Uploading \"" << convertedScoreName << "\" for conversion (type: " << convertTypeToString(type) << ")";
312 :
313 53 : const int uploadId = ++m_lastUploadId;
314 53 : m_uploadingScores.push_back(UploadingScore { uploadId, convertedScoreName });
315 53 : m_uploadProgresses[uploadId] = uploadProgress;
316 53 : m_uploadingScoresChanged.notify();
317 :
318 53 : Progress progress;
319 53 : progress.start();
320 :
321 53 : uploadProgress->progressChanged().onReceive(this, [convertedScoreName, progress](int64_t current, int64_t total,
322 : const std::string& msg) mutable {
323 1 : LOGI() << "Uploading \"" << convertedScoreName << "\" for conversion: " << current << "/" << total;
324 1 : progress.progress(current, total, msg);
325 1 : });
326 :
327 53 : uploadProgress->finished().onReceive(this, [this, type, uploadId, convertedScoreName, progress](const ProgressResult& res) mutable {
328 52 : removeUpload(uploadId);
329 :
330 52 : if (res.ret.code() == int(Ret::Code::Cancel)) {
331 1 : LOGI() << "Upload of \"" << convertedScoreName << "\" was canceled";
332 1 : progress.finish(res.ret);
333 1 : return;
334 : }
335 :
336 51 : if (!res.ret) {
337 4 : LOGE() << "Could not upload files for \"" << convertedScoreName << "\" (type: "
338 2 : << convertTypeToString(type) << "): " << res.ret.toString();
339 :
340 2 : const ConvertErrorCode errorCode = res.ret.data<ConvertErrorCode>(CONVERT_ERROR_CODE_KEY, ConvertErrorCode::Unknown);
341 2 : const Err err = errorCode == ConvertErrorCode::RateLimited ? Err::ConvertRateLimited : Err::ConvertUploadFailed;
342 2 : progress.finish(make_ret(err, res.ret.toString()));
343 2 : return;
344 : }
345 :
346 147 : const int itemId = res.val.toMap()["id"].toInt();
347 49 : watch(type, itemId, convertedScoreName);
348 49 : progress.finish(make_ok());
349 : });
350 :
351 53 : return RetVal<Progress>::make_ok(progress);
352 54 : }
353 :
354 11 : async::Channel<Ret, WatchedScore> ConvertFileToScoreService::convertFinished() const
355 : {
356 11 : return m_convertFinished;
357 : }
358 :
359 12 : ValNt<UploadingScoreList> ConvertFileToScoreService::uploadingScores() const
360 : {
361 12 : ValNt<UploadingScoreList> result;
362 12 : result.val = m_uploadingScores;
363 12 : result.notification = m_uploadingScoresChanged;
364 :
365 12 : return result;
366 0 : }
367 :
368 27 : ValNt<WatchedScoreList> ConvertFileToScoreService::watchedScores() const
369 : {
370 27 : ValNt<WatchedScoreList> result;
371 27 : result.val = m_watchedScores;
372 27 : result.notification = m_watchedScoresChanged;
373 :
374 27 : return result;
375 0 : }
376 :
377 3 : const WatchedScore* ConvertFileToScoreService::watchedScoreById(int scoreId) const
378 : {
379 3 : auto it = std::find_if(m_watchedScores.cbegin(), m_watchedScores.cend(), [scoreId](const WatchedScore& watched) {
380 3 : return watched.scoreId == scoreId;
381 : });
382 :
383 3 : return it != m_watchedScores.cend() ? &*it : nullptr;
384 : }
385 :
386 6 : async::Channel<PollingStatus> ConvertFileToScoreService::pollingStatusChanged() const
387 : {
388 6 : return m_pollingStatusChanged;
389 : }
390 :
391 0 : int64_t ConvertFileToScoreService::nowMs() const
392 : {
393 0 : return std::chrono::duration_cast<std::chrono::milliseconds>(
394 0 : std::chrono::steady_clock::now().time_since_epoch()).count();
395 : }
396 :
397 4 : void ConvertFileToScoreService::retryPolling()
398 : {
399 4 : resetPollState();
400 4 : m_manualRetryRequested = true;
401 4 : m_timer.start();
402 4 : poll();
403 4 : }
404 :
405 2 : void ConvertFileToScoreService::submitReview(int scoreId, ReviewRating rating, const QString& comment)
406 : {
407 2 : IF_ASSERT_FAILED(rating == ReviewRating::Bad || comment.isEmpty()) {
408 0 : return;
409 : }
410 :
411 2 : const WatchedScore* watched = watchedScoreById(scoreId);
412 2 : IF_ASSERT_FAILED(watched) {
413 0 : return;
414 : }
415 :
416 2 : const ConvertType type = watched->conversion.type;
417 2 : const int convertId = watched->conversion.id;
418 :
419 4 : museScoreComService()->convert()->submitReview(type, convertId, rating, comment)
420 2 : .onResolve(this, [this, type, convertId](const RetVal<ConvertResult>& submitRes) {
421 2 : if (!submitRes.ret) {
422 0 : LOGE() << "Could not submit the review for conversion ("
423 0 : << convertIdAndType(type, convertId) << "): " << submitRes.ret.toString();
424 0 : return;
425 : }
426 :
427 2 : const auto it = std::find_if(m_watchedScores.begin(), m_watchedScores.end(), [type, convertId](const WatchedScore& watched) {
428 2 : return watched.conversion.type == type && watched.conversion.id == convertId;
429 : });
430 :
431 2 : if (it == m_watchedScores.end()) {
432 0 : return;
433 : }
434 :
435 2 : updateStatus(*it, submitRes.val.status, ConvertErrorCode::Unknown);
436 :
437 2 : if (it->conversion.status == ConvertStatus::Done) {
438 1 : m_watchedScores.erase(it);
439 : }
440 :
441 2 : saveWatchedScores();
442 2 : m_watchedScoresChanged.notify();
443 : });
444 : }
445 :
446 1 : void ConvertFileToScoreService::submitReviewComment(int scoreId, const QString& comment)
447 : {
448 1 : const WatchedScore* watched = watchedScoreById(scoreId);
449 1 : IF_ASSERT_FAILED(watched) {
450 0 : return;
451 : }
452 :
453 1 : const ConvertType type = watched->conversion.type;
454 1 : const int convertId = watched->conversion.id;
455 :
456 2 : museScoreComService()->convert()->submitReviewComment(type, convertId, comment)
457 1 : .onResolve(this, [type, convertId](const Ret& ret) {
458 1 : if (!ret) {
459 0 : LOGE() << "Could not submit the comment for conversion ("
460 0 : << convertIdAndType(type, convertId) << "): " << ret.toString();
461 : }
462 1 : });
463 : }
464 :
465 1 : void ConvertFileToScoreService::cancelUpload(int uploadId)
466 : {
467 1 : auto it = m_uploadProgresses.find(uploadId);
468 1 : if (it == m_uploadProgresses.end()) {
469 0 : return;
470 : }
471 :
472 : //! NOTE: the upload's finished() handler removes it
473 1 : ProgressPtr progress = it->second;
474 1 : progress->cancel();
475 1 : }
476 :
477 2 : void ConvertFileToScoreService::cancelConversion(ConvertType type, int convertId)
478 : {
479 4 : museScoreComService()->convert()->deleteConversion(type, convertId)
480 2 : .onResolve(this, [this, type, convertId](const Ret& ret) {
481 2 : if (!ret) {
482 1 : LOGE() << "Could not delete conversion (" << convertIdAndType(type, convertId) << "): " << ret.toString();
483 1 : return;
484 : }
485 :
486 1 : const auto it = std::find_if(m_watchedScores.begin(), m_watchedScores.end(), [type, convertId](const WatchedScore& watched) {
487 1 : return watched.conversion.type == type && watched.conversion.id == convertId;
488 : });
489 :
490 1 : if (it == m_watchedScores.end()) {
491 0 : return;
492 : }
493 :
494 1 : m_watchedScores.erase(it);
495 1 : saveWatchedScores();
496 1 : m_watchedScoresChanged.notify();
497 : });
498 2 : }
499 :
500 4 : void ConvertFileToScoreService::loadWatchedScores()
501 : {
502 5 : TRACEFUNC;
503 :
504 4 : RetVal<ByteArray> data;
505 : {
506 4 : muse::mi::ReadResourceLockGuard resource_guard(multiwindowsProvider(), WATCHED_CONVERTS_RESOURCE_NAME);
507 4 : data = fileSystem()->readFile(configuration()->watchedConvertsJsonPath());
508 4 : }
509 :
510 4 : if (!data.ret || data.val.empty()) {
511 0 : if (!data.ret && data.ret.code() != static_cast<int>(io::Err::FSNotExist)) {
512 0 : LOGE() << "Could not read the pending conversions file: " << data.ret;
513 : }
514 0 : return;
515 : }
516 :
517 4 : std::string err;
518 4 : const JsonDocument json = JsonDocument::fromJson(data.val, &err);
519 4 : if (!err.empty() || !json.isArray()) {
520 0 : if (!err.empty()) {
521 0 : LOGE() << "Could not parse the pending conversions file: " << err;
522 : }
523 0 : return;
524 : }
525 :
526 4 : const JsonArray array = json.rootArray();
527 4 : m_watchedScores.clear();
528 4 : m_watchedScores.reserve(array.size());
529 :
530 10 : for (size_t i = 0; i < array.size(); ++i) {
531 6 : const JsonObject obj = array.at(i).toObject();
532 18 : const int typeInt = obj.value("type").toInt();
533 6 : if (typeInt < 0 || typeInt > static_cast<int>(ConvertType::Last)) {
534 0 : LOGW() << "Skipping conversion with unknown type: " << typeInt;
535 0 : continue;
536 : }
537 :
538 18 : int statusInt = obj.value("status").toInt();
539 6 : if (statusInt < 0 || statusInt > static_cast<int>(ConvertStatus::Unknown)) {
540 1 : LOGW() << "Unknown conversion status: " << statusInt;
541 1 : statusInt = static_cast<int>(ConvertStatus::Unknown);
542 : }
543 :
544 6 : WatchedScore& watched = m_watchedScores.emplace_back();
545 18 : watched.conversion.id = obj.value("id").toInt();
546 6 : watched.conversion.type = static_cast<ConvertType>(typeInt);
547 6 : watched.conversion.status = static_cast<ConvertStatus>(statusInt);
548 6 : watched.scoreId = obj.contains("scoreId") ? std::optional<int>(obj.value("scoreId").toInt()) : std::nullopt;
549 18 : watched.startedLocally = obj.value("startedLocally").toBool();
550 18 : watched.name = muse::String::fromStdString(obj.value("convertedScoreName").toStdString());
551 6 : }
552 4 : }
553 :
554 69 : void ConvertFileToScoreService::saveWatchedScores()
555 : {
556 70 : TRACEFUNC;
557 :
558 69 : JsonArray array;
559 161 : for (const WatchedScore& watched : m_watchedScores) {
560 92 : JsonObject obj;
561 184 : obj["id"] = watched.conversion.id;
562 184 : obj["type"] = static_cast<int>(watched.conversion.type);
563 184 : obj["status"] = static_cast<int>(watched.conversion.status);
564 184 : obj["startedLocally"] = watched.startedLocally;
565 :
566 92 : if (!watched.name.isEmpty()) {
567 276 : obj["convertedScoreName"] = watched.name.toStdString();
568 : }
569 :
570 92 : if (watched.scoreId) {
571 24 : obj["scoreId"] = *watched.scoreId;
572 : }
573 :
574 92 : array << obj;
575 92 : }
576 :
577 69 : JsonDocument json(array);
578 :
579 69 : m_isSaving = true;
580 : {
581 69 : muse::mi::WriteResourceLockGuard resource_guard(multiwindowsProvider(), WATCHED_CONVERTS_RESOURCE_NAME);
582 69 : Ret ret = fileSystem()->writeFile(configuration()->watchedConvertsJsonPath(), json.toJson());
583 69 : if (!ret) {
584 65 : LOGE() << "Could not save the pending conversions list: " << ret.toString();
585 : }
586 69 : }
587 69 : m_isSaving = false;
588 69 : }
589 :
590 52 : void ConvertFileToScoreService::removeUpload(int uploadId)
591 : {
592 52 : m_uploadProgresses.erase(uploadId);
593 :
594 52 : auto it = std::find_if(m_uploadingScores.begin(), m_uploadingScores.end(), [uploadId](const UploadingScore& uploading) {
595 52 : return uploading.uploadId == uploadId;
596 : });
597 52 : if (it != m_uploadingScores.end()) {
598 52 : m_uploadingScores.erase(it);
599 52 : m_uploadingScoresChanged.notify();
600 : }
601 52 : }
602 :
603 49 : void ConvertFileToScoreService::watch(ConvertType type, int itemId, const muse::String& convertedScoreName)
604 : {
605 49 : LOGI() << "Start watching conversion of \"" << convertedScoreName << "\" (" << convertIdAndType(type, itemId) << ")";
606 :
607 49 : const auto it = std::find_if(m_watchedScores.begin(), m_watchedScores.end(), [type, itemId](const WatchedScore& watched) {
608 33 : return watched.conversion.type == type && watched.conversion.id == itemId;
609 : });
610 49 : WatchedScore& watched = it != m_watchedScores.end() ? *it : m_watchedScores.emplace_back();
611 49 : watched.conversion.id = itemId;
612 49 : watched.conversion.type = type;
613 49 : watched.conversion.status = ConvertStatus::Processing;
614 49 : watched.startedLocally = true;
615 49 : watched.name = convertedScoreName;
616 :
617 49 : saveWatchedScores();
618 49 : m_watchedScoresChanged.notify();
619 :
620 49 : if (!m_timer.isActive()) {
621 28 : m_timer.start();
622 : }
623 49 : }
624 :
625 51 : bool ConvertFileToScoreService::hasAnyScoresToPoll() const
626 : {
627 51 : return std::any_of(m_watchedScores.begin(), m_watchedScores.end(), [](const WatchedScore& watched) {
628 52 : return watched.conversion.status == ConvertStatus::Processing
629 52 : || watched.conversion.status == ConvertStatus::Unknown;
630 51 : });
631 : }
632 :
633 51 : void ConvertFileToScoreService::poll()
634 : {
635 51 : if (!hasAnyScoresToPoll()) {
636 3 : LOGDA() << "Nothing active to poll, stopping timer";
637 3 : m_timer.stop();
638 3 : m_manualRetryRequested = false;
639 3 : return;
640 : }
641 :
642 48 : if (m_pollInProgress) {
643 0 : LOGDA() << "Polling already in progress, skipping";
644 0 : return;
645 : }
646 :
647 48 : m_pollInProgress = true;
648 :
649 : //! NOTE: Take a snapshot of what we're checking
650 : //! Items added to m_watchedScores while this request is running
651 : //! were not part of it, so don't compare them with the response
652 48 : const WatchedScoreList snapshot = m_watchedScores;
653 :
654 48 : museScoreComService()->convert()->fetchQueue().onResolve(this, [this, snapshot](const RetVal<ConvertQueueList>& result) {
655 44 : m_pollInProgress = false;
656 :
657 44 : if (!result.ret) {
658 22 : handlePollFailure(result.ret);
659 22 : return;
660 : }
661 :
662 22 : resetPollState();
663 22 : m_pollingStatusChanged.send(PollingSuccess {});
664 22 : updateWatchedScores(result.val, snapshot);
665 : });
666 48 : }
667 :
668 30 : void ConvertFileToScoreService::resetPollState()
669 : {
670 30 : m_pollFailureCount = 0;
671 30 : m_pollIntervalMs = MIN_RETRY_INTERVAL_MS;
672 30 : m_timer.setInterval(MIN_RETRY_INTERVAL_MS);
673 30 : m_manualRetryRequested = false;
674 30 : }
675 :
676 22 : void ConvertFileToScoreService::handlePollFailure(const Ret& ret)
677 : {
678 22 : if (m_pollFailureCount == 0) {
679 9 : m_pollRetryStartMs = nowMs();
680 : }
681 22 : ++m_pollFailureCount;
682 :
683 22 : const int64_t elapsedMs = nowMs() - m_pollRetryStartMs;
684 22 : const int64_t remainingMs = MAX_POLL_RETRY_DURATION_MS - elapsedMs;
685 22 : const secs_t elapsedSecs(elapsedMs / 1000.0);
686 :
687 22 : if (!m_manualRetryRequested && isRetryableError(ret) && remainingMs > 0) {
688 18 : m_pollIntervalMs = static_cast<int>(std::min<int64_t>(m_pollIntervalMs * 2, remainingMs));
689 18 : m_timer.setInterval(m_pollIntervalMs);
690 18 : const secs_t intervalSecs(m_pollIntervalMs / 1000.0);
691 36 : LOGW() << "Could not check the conversion status, retrying in " << intervalSecs
692 18 : << "s (elapsed " << elapsedSecs << "s): " << ret.toString();
693 18 : m_pollingStatusChanged.send(PollingFailure { ret, elapsedSecs, intervalSecs, false });
694 18 : return;
695 : }
696 :
697 4 : giveUpPolling(ret, elapsedSecs);
698 : }
699 :
700 4 : void ConvertFileToScoreService::giveUpPolling(const Ret& ret, secs_t elapsed)
701 : {
702 8 : LOGE() << "Could not check the conversion status, stopping polling for now, "
703 4 : << m_watchedScores.size() << " pending conversion(s) remain watched: " << ret.toString();
704 :
705 4 : m_timer.stop();
706 4 : resetPollState();
707 :
708 4 : m_pollingStatusChanged.send(PollingFailure { ret, elapsed, secs_t(0), true });
709 4 : }
710 :
711 22 : void ConvertFileToScoreService::updateWatchedScores(const ConvertQueueList& queue, const WatchedScoreList& snapshot)
712 : {
713 23 : TRACEFUNC;
714 :
715 22 : LOGDA() << "Received queue: " << queue.size() << ", snapshot: " << snapshot.size();
716 :
717 22 : constexpr size_t CONVERT_TYPE_COUNT = static_cast<size_t>(ConvertType::Last) + 1;
718 22 : std::array<std::unordered_map<int /*itemId*/, size_t /*index*/>, CONVERT_TYPE_COUNT> snapshotByTypeAndId;
719 47 : for (size_t i = 0; i < snapshot.size(); ++i) {
720 25 : const WatchedScore& watched = snapshot[i];
721 25 : snapshotByTypeAndId[static_cast<size_t>(watched.conversion.type)][watched.conversion.id] = i;
722 : }
723 :
724 22 : std::vector<bool> seenInSnapshot(snapshot.size(), false);
725 :
726 : //! NOTE: m_watchedScores also covers items watched while a poll is in progress
727 22 : std::array<std::unordered_map<int /*itemId*/, size_t /*index*/>, CONVERT_TYPE_COUNT> liveByTypeAndId;
728 51 : for (size_t i = 0; i < m_watchedScores.size(); ++i) {
729 29 : const WatchedScore& watched = m_watchedScores[i];
730 29 : liveByTypeAndId[static_cast<size_t>(watched.conversion.type)][watched.conversion.id] = i;
731 : }
732 :
733 22 : std::vector<bool> seenInWatchedScores(m_watchedScores.size(), false);
734 :
735 : //! NOTE: the queue is the source of truth - rebuild m_watchedScores from it every time,
736 : //! since it may also contain conversions started outside MuseScore
737 22 : std::vector<WatchedScore> newWatchedScores;
738 22 : newWatchedScores.reserve(queue.size());
739 :
740 49 : for (const ConvertQueueItem& queueItem : queue) {
741 27 : if (queueItem.type == ConvertType::Unknown) {
742 0 : LOGW() << "Skipping queue item with unrecognized type (" << convertIdAndType(queueItem.type, queueItem.id) << ")";
743 0 : continue;
744 : }
745 :
746 27 : const std::unordered_map<int, size_t>& liveIndexById = liveByTypeAndId[static_cast<size_t>(queueItem.type)];
747 27 : const auto liveIt = liveIndexById.find(queueItem.id);
748 :
749 27 : if (liveIt == liveIndexById.end()
750 27 : && queueItem.status != ConvertStatus::Processing && queueItem.status != ConvertStatus::AwaitingReview) {
751 : //! NOTE: not previously watched, and already terminal - nothing to watch for anymore
752 0 : continue;
753 : }
754 :
755 27 : WatchedScore watched;
756 27 : if (liveIt != liveIndexById.end()) {
757 : //! NOTE: already watched - whether known before the poll started, or watched mid-poll
758 26 : watched = m_watchedScores.at(liveIt->second);
759 26 : seenInWatchedScores[liveIt->second] = true;
760 : } else {
761 1 : LOGI() << "Found new external conversion (" << convertIdAndType(queueItem.type, queueItem.id) << ")";
762 : }
763 :
764 27 : const std::unordered_map<int, size_t>& snapshotIndexById = snapshotByTypeAndId[static_cast<size_t>(queueItem.type)];
765 27 : const auto snapshotIt = snapshotIndexById.find(queueItem.id);
766 27 : if (snapshotIt != snapshotIndexById.end()) {
767 23 : seenInSnapshot[snapshotIt->second] = true;
768 : }
769 :
770 27 : watched.conversion.id = queueItem.id;
771 27 : watched.conversion.type = queueItem.type;
772 27 : if (!queueItem.filename.isEmpty()) {
773 5 : watched.name = queueItem.filename;
774 : }
775 27 : watched.scoreId = queueItem.scoreId;
776 :
777 27 : updateStatus(watched, queueItem.status, queueItem.errorCode);
778 :
779 27 : if (watched.conversion.status != ConvertStatus::Done) {
780 22 : newWatchedScores.push_back(watched);
781 : }
782 27 : }
783 :
784 47 : for (size_t i = 0; i < snapshot.size(); ++i) {
785 25 : if (seenInSnapshot.at(i)) {
786 23 : continue;
787 : }
788 :
789 2 : WatchedScore dropped = snapshot.at(i);
790 :
791 2 : if (dropped.scoreId) {
792 2 : LOGI() << "Conversion of \"" << dropped.name << "\" (" << convertIdAndType(dropped.conversion.type, dropped.conversion.id)
793 1 : << ") was dropped from the queue, recovering as Done with scoreId " << *dropped.scoreId;
794 :
795 : //! NOTE: always terminal, so never added back to newWatchedScores
796 1 : updateStatus(dropped, ConvertStatus::Done, ConvertErrorCode::Unknown);
797 : } else {
798 2 : LOGW() << "Conversion of \"" << dropped.name << "\" (" << convertIdAndType(dropped.conversion.type, dropped.conversion.id)
799 1 : << ") was dropped from the queue without ever reporting a scoreId";
800 : }
801 2 : }
802 :
803 : //! NOTE: items watched mid-poll that never appeared in the queue response at all (the server
804 : //! hasn't listed them yet) - carry them over untouched
805 51 : for (size_t i = 0; i < m_watchedScores.size(); ++i) {
806 29 : if (seenInWatchedScores[i]) {
807 26 : continue;
808 : }
809 :
810 3 : const WatchedScore& watched = m_watchedScores[i];
811 3 : const std::unordered_map<int, size_t>& snapshotIndexById = snapshotByTypeAndId[static_cast<size_t>(watched.conversion.type)];
812 3 : if (snapshotIndexById.find(watched.conversion.id) != snapshotIndexById.end()) {
813 2 : continue;
814 : }
815 :
816 1 : newWatchedScores.push_back(watched);
817 : }
818 :
819 22 : if (m_watchedScores != newWatchedScores) {
820 17 : m_watchedScores = std::move(newWatchedScores);
821 17 : saveWatchedScores();
822 17 : m_watchedScoresChanged.notify();
823 : }
824 22 : }
825 :
826 30 : void ConvertFileToScoreService::updateStatus(WatchedScore& watched, ConvertStatus newStatus, ConvertErrorCode errorCode)
827 : {
828 30 : const ConvertStatus previousStatus = watched.conversion.status;
829 30 : if (previousStatus == newStatus) {
830 11 : return;
831 : }
832 :
833 38 : LOGI() << "Conversion of \"" << watched.name << "\" (" << convertIdAndType(watched.conversion.type, watched.conversion.id) << ")"
834 19 : << " status changed: " << convertStatusToString(previousStatus) << " -> " << convertStatusToString(newStatus);
835 :
836 19 : const bool wasDone = previousStatus == ConvertStatus::Done
837 19 : || previousStatus == ConvertStatus::AwaitingReview;
838 :
839 19 : watched.conversion.status = newStatus;
840 :
841 19 : switch (newStatus) {
842 2 : case ConvertStatus::Processing:
843 : case ConvertStatus::Unknown:
844 2 : break;
845 14 : case ConvertStatus::AwaitingReview:
846 : case ConvertStatus::Done:
847 14 : if (!wasDone && watched.scoreId && watched.startedLocally) {
848 11 : m_convertFinished.send(make_ok(), watched);
849 : }
850 14 : break;
851 3 : case ConvertStatus::Failed: {
852 3 : Ret ret = make_ret(Err::ConvertProcessingFailed);
853 3 : ret.setText("Conversion failed for \"" + watched.name.toStdString() + "\": " + errorCodeToString(errorCode));
854 :
855 3 : LOGE() << ret.toString();
856 :
857 3 : if (watched.startedLocally) {
858 3 : m_convertFinished.send(ret, watched);
859 : }
860 3 : break;
861 3 : }
862 : }
863 : }
|
