fix: a game build is recognised before the archive integrity test, so re-extract no longer stalls on it
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / frontend-build (push) Successful in 24s
CI and images / extension-test (push) Successful in 18s
CI and images / backend-lint-and-test (push) Successful in 47s
CI and images / integration (push) Successful in 2m44s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 2m1s
CI and images / smoke-web (push) Successful in 49s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / frontend-build (push) Successful in 24s
CI and images / extension-test (push) Successful in 18s
CI and images / backend-lint-and-test (push) Successful in 47s
CI and images / integration (push) Successful in 2m44s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 2m1s
CI and images / smoke-web (push) Successful in 49s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
The archive probe read and checksummed every member before the importer looked at what the archive held. For a multi-gigabyte game build that is minutes of work on a file that is kept whole and never extracted, and it held a re-extract chunk far past its time budget. - The importer checks the archive's directory for a software build first, for the outer archive and for one nested inside an art pack, and only probes archives it may extract. - The probe rejects an archive over the size cap from its directory totals, before the integrity test decompresses anything. - Re-extract logs each archive as it starts and finishes, with its size and how long it took, so an archive that stalls a chunk is named in the log. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1205,6 +1205,11 @@ def reextract_archive_attachments(
|
||||
)
|
||||
if post is None:
|
||||
summary["without_post"] += 1
|
||||
# Logged on the way in as well as out, so an archive that stalls a
|
||||
# chunk is named in the log even if the worker dies holding it.
|
||||
size_mb = (att.size_bytes or 0) / 1e6
|
||||
log.info("re-extract: attachment %s (%.0f MB) starting", att.id, size_mb)
|
||||
archive_start = time.monotonic()
|
||||
try:
|
||||
ids = _reextract_archive_to_post(
|
||||
importer, stored, post, source_row, artist, images_root,
|
||||
@@ -1214,8 +1219,11 @@ def reextract_archive_attachments(
|
||||
except Exception as exc: # one bad archive must not strand the rest
|
||||
session.rollback()
|
||||
summary["errors"] += 1
|
||||
log.warning("re-extract failed for attachment %s: %s", att.id, exc)
|
||||
log.warning("re-extract failed for attachment %s after %.1fs: %s",
|
||||
att.id, time.monotonic() - archive_start, exc)
|
||||
continue
|
||||
log.info("re-extract: attachment %s done in %.1fs, %d new image(s)",
|
||||
att.id, time.monotonic() - archive_start, len(ids))
|
||||
if ids:
|
||||
summary["members_imported"] += len(ids)
|
||||
if post is not None:
|
||||
|
||||
@@ -594,29 +594,22 @@ class Importer:
|
||||
# lays files out under /images/<artist_slug>/...). Filesystem
|
||||
# import still calls bare _import_archive(source) and falls
|
||||
# back to the path-walk derivation as before.
|
||||
probe = safe_probe.probe_archive(source)
|
||||
if not probe.ok:
|
||||
if probe.crashed:
|
||||
return ImportResult(
|
||||
status="failed",
|
||||
error=f"archive probe crashed/timed out: {probe.reason}",
|
||||
)
|
||||
# Clean rejection (bomb cap exceeded, integrity mismatch):
|
||||
# still preserve the archive file itself as an attachment so
|
||||
# nothing silently vanishes, matching extract_archive's
|
||||
# fail-soft contract.
|
||||
artist_use = artist if artist is not None else self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist_use, source_row=source_row)
|
||||
self._capture_attachment(
|
||||
source, post=post, artist=artist_use, resolved=True,
|
||||
#
|
||||
# A game or app build is recognised first, from the archive's directory
|
||||
# alone. The probe reads and checksums every member, which for a
|
||||
# multi-gigabyte build is minutes of work on a file that is kept whole
|
||||
# and never extracted anyway.
|
||||
marker = software_marker(source)
|
||||
probe = None if marker is not None else safe_probe.probe_archive(source)
|
||||
if probe is not None and probe.crashed:
|
||||
return ImportResult(
|
||||
status="failed",
|
||||
error=f"archive probe crashed/timed out: {probe.reason}",
|
||||
)
|
||||
reason = f"archive probe rejected, captured unextracted: {probe.reason}"
|
||||
log.warning("%s: %s", source.name, reason)
|
||||
return ImportResult(status="attached", error=reason)
|
||||
|
||||
artist_use = artist if artist is not None else self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist_use, source_row=source_row)
|
||||
if (marker := software_marker(source)) is not None:
|
||||
if marker is not None:
|
||||
# A game or app build: kept whole, its bundled images left alone.
|
||||
self._capture_attachment(
|
||||
source, post=post, artist=artist_use, resolved=True,
|
||||
@@ -624,6 +617,16 @@ class Importer:
|
||||
log.info("%s: a software build (holds %s), kept unextracted",
|
||||
source.name, marker)
|
||||
return ImportResult(status="attached")
|
||||
if not probe.ok:
|
||||
# Clean rejection (bomb cap exceeded, integrity mismatch): the
|
||||
# archive file itself is still kept as an attachment so nothing
|
||||
# silently vanishes, matching extract_archive's fail-soft contract.
|
||||
self._capture_attachment(
|
||||
source, post=post, artist=artist_use, resolved=True,
|
||||
)
|
||||
reason = f"archive probe rejected, captured unextracted: {probe.reason}"
|
||||
log.warning("%s: %s", source.name, reason)
|
||||
return ImportResult(status="attached", error=reason)
|
||||
# Committed before extraction, so a member that fails (and is rolled
|
||||
# back on its own) can never take the archive's post with it.
|
||||
self.session.commit()
|
||||
@@ -718,6 +721,15 @@ class Importer:
|
||||
_ARCHIVE_MAX_DEPTH, member_path.name,
|
||||
)
|
||||
continue
|
||||
# Recognised from its directory before the probe reads
|
||||
# every member, as for the outer archive.
|
||||
if (marker := software_marker(member_path)) is not None:
|
||||
counts["nested_software"] += 1
|
||||
log.info(
|
||||
"nested archive is a software build (holds %s), "
|
||||
"skipped: %s", marker, member_path.name,
|
||||
)
|
||||
continue
|
||||
probe = safe_probe.probe_archive(member_path)
|
||||
if not probe.ok:
|
||||
counts["nested_rejected"] += 1
|
||||
@@ -726,13 +738,6 @@ class Importer:
|
||||
probe.reason, member_path.name,
|
||||
)
|
||||
continue
|
||||
if (marker := software_marker(member_path)) is not None:
|
||||
counts["nested_software"] += 1
|
||||
log.info(
|
||||
"nested archive is a software build (holds %s), "
|
||||
"skipped: %s", marker, member_path.name,
|
||||
)
|
||||
continue
|
||||
self._collect_archive_members(
|
||||
member_path, attribution=attribution,
|
||||
source_row=source_row, depth=depth + 1,
|
||||
|
||||
Reference in New Issue
Block a user