Skip to content

Atomic conversion outputs #28

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
anjackson wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Atomic conversion outputs #28

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b6c88c5
Add checks for converion outputs, atomic updates.
anjackson Oct 3, 2023
1ce50e6
Use tmp prefix so converters can recognise the file extension.
anjackson Oct 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 23 additions & 4 deletions warcit/converter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ def main(args=None):

parser.add_argument('--dry-run', action='store_true')

parser.add_argument('--overwrite', action='store_true', help="Overwrite output files")

parser.add_argument('--output-dir', help='Root output directory for conversions')

parser.add_argument('-q', '--quiet', action='store_true')
Expand Down Expand Up @@ -57,7 +59,7 @@ def main(args=None):
output_dir=r.output_dir,
results_file=r.results)

converter.convert_all(dry_run=r.dry_run)
converter.convert_all(dry_run=r.dry_run,overwrite=r.overwrite)


# ============================================================================
Expand Down Expand Up @@ -114,7 +116,7 @@ def write_results(self):
with open(filename, 'wt') as fh:
fh.write(yaml.dump(root, default_flow_style=False))

def convert_all(self, dry_run=False):
def convert_all(self, dry_run=False, overwrite=False):
stdout = None
if self.convert_stdout:
stdout = open(self.convert_stdout, 'wt')
Expand All @@ -124,7 +126,8 @@ def convert_all(self, dry_run=False):
self.convert_file(file_info,
dry_run=dry_run,
convert_stdout=stdout,
convert_stderr=stdout)
convert_stderr=stdout,
overwrite=overwrite)

if not dry_run:
self.write_results()
Expand All @@ -133,7 +136,7 @@ def convert_all(self, dry_run=False):
if stdout:
stdout.close()

def convert_file(self, file_info, dry_run=False, convert_stdout=None, convert_stderr=None):
def convert_file(self, file_info, dry_run=False, convert_stdout=None, convert_stderr=None, overwrite=False):
for file_type in self.file_types:
matched = False
# first, check by extension if available
Expand All @@ -155,6 +158,18 @@ def convert_file(self, file_info, dry_run=False, convert_stdout=None, convert_st
root_dir=file_info.root_dir)

self.logger.debug('Output Filename: ' + output)

# Do not overwrite existing files:
if os.path.exists(output) and not overwrite:
self.logger.warning(f"Path {output} already exists! Skipping {conversion['name']}")
continue

# Define a temporary name so updates will be atomic
original_output = output
tmp_name = f"tmp_{os.path.basename(output)}"
output = os.path.join(os.path.dirname(output), tmp_name)

# Set up the command to run:
command = conversion['command'].format(input=file_info.full_filename,
output=output)

Expand All @@ -168,6 +183,10 @@ def convert_file(self, file_info, dry_run=False, convert_stdout=None, convert_st

self.logger.debug('Exit Code: {0}'.format(res))

# Move the _tmp file to the final filename:
os.rename(output, original_output)
output = original_output

result = {'url': file_info.url + '.' + conversion['ext'],
'output': output,
'metadata': conversion,
Expand Down