diff --git a/mapillary_tools/commands/__init__.py b/mapillary_tools/commands/__init__.py index 7e7e4dc49..2a4fe2596 100644 --- a/mapillary_tools/commands/__init__.py +++ b/mapillary_tools/commands/__init__.py @@ -5,7 +5,6 @@ process_and_upload, sample_video, upload, - upload_camm, video_process, video_process_and_upload, ) diff --git a/mapillary_tools/commands/__main__.py b/mapillary_tools/commands/__main__.py index 884132a06..b9f0ca226 100644 --- a/mapillary_tools/commands/__main__.py +++ b/mapillary_tools/commands/__main__.py @@ -12,9 +12,6 @@ process_and_upload, sample_video, upload, - upload_blackvue, - upload_camm, - upload_zip, video_process, video_process_and_upload, zip, @@ -23,9 +20,6 @@ mapillary_tools_commands = [ process, upload, - upload_camm, - upload_blackvue, - upload_zip, sample_video, video_process, authenticate, diff --git a/mapillary_tools/commands/process.py b/mapillary_tools/commands/process.py index c93d57fb2..239b64510 100644 --- a/mapillary_tools/commands/process.py +++ b/mapillary_tools/commands/process.py @@ -107,18 +107,6 @@ def add_basic_arguments(self, parser: argparse.ArgumentParser): default=None, required=False, ) - group_metadata.add_argument( - "--add_file_name", - help="[DEPRECATED since v0.9.4] Add original file name to EXIF.", - action="store_true", - required=False, - ) - group_metadata.add_argument( - "--add_import_date", - help="[DEPRECATED since v0.10.0] Add import date.", - action="store_true", - required=False, - ) group_metadata.add_argument( "--orientation", help="Specify the image orientation in degrees. Note this might result in image rotation. Note this input has precedence over the input read from the import source file.", @@ -139,12 +127,6 @@ def add_basic_arguments(self, parser: argparse.ArgumentParser): default=None, required=False, ) - group_metadata.add_argument( - "--custom_meta_data", - help='[DEPRECATED since v0.10.0] Add custom meta data to all images. Required format of input is a string, consisting of the meta data name, type and value, separated by a comma for each entry, where entries are separated by semicolon. Supported types are long, double, string, boolean, date. Example for two meta data entries "random_name1,double,12.34;random_name2,long,1234".', - default=None, - required=False, - ) group_geotagging = parser.add_argument_group( f"{constants.ANSI_BOLD}PROCESS GEOTAGGING OPTIONS{constants.ANSI_RESET_ALL}" diff --git a/mapillary_tools/commands/upload.py b/mapillary_tools/commands/upload.py index 67e26d2f9..34cfc1d3f 100644 --- a/mapillary_tools/commands/upload.py +++ b/mapillary_tools/commands/upload.py @@ -1,8 +1,7 @@ import inspect -import typing as T from .. import constants -from ..upload import DirectUploadFileType, FileType, upload +from ..upload import upload class Command: @@ -34,30 +33,6 @@ def add_basic_arguments(self, parser): group = parser.add_argument_group( f"{constants.ANSI_BOLD}UPLOAD OPTIONS{constants.ANSI_RESET_ALL}" ) - default_filetypes = ",".join(sorted(t.value for t in FileType)) - supported_filetypes = ",".join( - sorted( - [t.value for t in DirectUploadFileType] + [t.value for t in FileType] - ) - ) - - def _type(option: str) -> T.List[T.Union[FileType, DirectUploadFileType]]: - r: T.List[T.Union[FileType, DirectUploadFileType]] = [] - for t in option.split(","): - if t in [x.value for x in FileType]: - r.append(FileType(t)) - else: - r.append(DirectUploadFileType(t)) - return r - - group.add_argument( - "--filetypes", - "--file_types", - help=f"Upload files of the specified types only. Supported file types: {supported_filetypes} [default: %(default)s]", - type=_type, - default=default_filetypes, - required=False, - ) group.add_argument( "--desc_path", help=f'Path to the description file generated by the process command. The hyphen "-" indicates STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]', diff --git a/mapillary_tools/commands/upload_blackvue.py b/mapillary_tools/commands/upload_blackvue.py deleted file mode 100644 index dd0397649..000000000 --- a/mapillary_tools/commands/upload_blackvue.py +++ /dev/null @@ -1,33 +0,0 @@ -import inspect -from pathlib import Path - -from .. import constants, upload -from .upload import Command as UploadCommand - - -class Command: - name = "upload_blackvue" - help = "[deprecated] upload BlackVue videos to Mapillary" - - def add_basic_arguments(self, parser): - parser.add_argument( - "import_path", - help="Path to your BlackVue videos.", - nargs="+", - type=Path, - ) - group = parser.add_argument_group( - f"{constants.ANSI_BOLD}UPLOAD OPTIONS{constants.ANSI_RESET_ALL}" - ) - UploadCommand.add_common_upload_options(group) - - def run(self, vars_args: dict): - args = { - k: v - for k, v in vars_args.items() - if k in inspect.getfullargspec(upload.upload).args - } - upload.upload( - **args, - filetypes={upload.DirectUploadFileType.RAW_BLACKVUE}, - ) diff --git a/mapillary_tools/commands/upload_camm.py b/mapillary_tools/commands/upload_camm.py deleted file mode 100644 index a58b9d374..000000000 --- a/mapillary_tools/commands/upload_camm.py +++ /dev/null @@ -1,33 +0,0 @@ -import inspect -from pathlib import Path - -from .. import constants, upload -from .upload import Command as UploadCommand - - -class Command: - name = "upload_camm" - help = "[deprecated] upload CAMM videos to Mapillary" - - def add_basic_arguments(self, parser): - parser.add_argument( - "import_path", - help="Path to your CAMM videos.", - nargs="+", - type=Path, - ) - group = parser.add_argument_group( - f"{constants.ANSI_BOLD}UPLOAD OPTIONS{constants.ANSI_RESET_ALL}" - ) - UploadCommand.add_common_upload_options(group) - - def run(self, vars_args: dict): - args = { - k: v - for k, v in vars_args.items() - if k in inspect.getfullargspec(upload.upload).args - } - upload.upload( - **args, - filetypes={upload.DirectUploadFileType.RAW_CAMM}, - ) diff --git a/mapillary_tools/commands/upload_zip.py b/mapillary_tools/commands/upload_zip.py deleted file mode 100644 index ca5463f04..000000000 --- a/mapillary_tools/commands/upload_zip.py +++ /dev/null @@ -1,33 +0,0 @@ -import inspect -from pathlib import Path - -from .. import constants, upload -from .upload import Command as UploadCommand - - -class Command: - name = "upload_zip" - help = "[deprecated] upload ZIP files to Mapillary" - - def add_basic_arguments(self, parser): - parser.add_argument( - "import_path", - help="Path to your ZIP files.", - nargs="+", - type=Path, - ) - group = parser.add_argument_group( - f"{constants.ANSI_BOLD}UPLOAD OPTIONS{constants.ANSI_RESET_ALL}" - ) - UploadCommand.add_common_upload_options(group) - - def run(self, vars_args: dict): - args = { - k: v - for k, v in vars_args.items() - if k in inspect.getfullargspec(upload.upload).args - } - upload.upload( - **args, - filetypes={upload.DirectUploadFileType.ZIP}, - ) diff --git a/mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py b/mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py index 2eb6b7e6e..c0837ade0 100644 --- a/mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py +++ b/mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py @@ -48,7 +48,6 @@ def to_description(self) -> T.List[types.ImageMetadataOrError]: video_paths = utils.find_videos( [Path(pathstr) for pathstr in rdf_description_by_path.keys()], skip_subfolders=True, - check_file_suffix=True, ) # will try to geotag these error metadatas from video later diff --git a/mapillary_tools/process_geotag_properties.py b/mapillary_tools/process_geotag_properties.py index ff6096f7f..bba2a33d1 100644 --- a/mapillary_tools/process_geotag_properties.py +++ b/mapillary_tools/process_geotag_properties.py @@ -206,16 +206,8 @@ def process_geotag_properties( metadatas: T.List[types.MetadataOrError] = [] - # if more than one filetypes speficied, check filename suffixes, - # i.e. files not ended with .jpg or .mp4 will be ignored - check_file_suffix = len(filetypes) > 1 - if FileType.IMAGE in filetypes: - image_paths = utils.find_images( - import_paths, - skip_subfolders=skip_subfolders, - check_file_suffix=check_file_suffix, - ) + image_paths = utils.find_images(import_paths, skip_subfolders=skip_subfolders) if image_paths: image_metadatas = _process_images( image_paths, @@ -240,9 +232,7 @@ def process_geotag_properties( or FileType.VIDEO in filetypes ): video_paths = utils.find_videos( - import_paths, - skip_subfolders=skip_subfolders, - check_file_suffix=check_file_suffix, + import_paths, skip_subfolders=skip_subfolders ) if video_paths: video_metadata = _process_videos( @@ -289,7 +279,6 @@ def _process_videos_beta(vars_args: T.Dict): "num_processes": vars_args["num_processes"], "device_make": vars_args["device_make"], "device_model": vars_args["device_model"], - "check_file_suffix": len(vars_args["filetypes"]) > 1, } extractor = VideoDataExtractor(options) return extractor.process() diff --git a/mapillary_tools/process_import_meta_properties.py b/mapillary_tools/process_import_meta_properties.py index a29b9c59a..bbbb0ee46 100644 --- a/mapillary_tools/process_import_meta_properties.py +++ b/mapillary_tools/process_import_meta_properties.py @@ -31,22 +31,8 @@ def process_import_meta_properties( device_make=None, device_model=None, GPS_accuracy=None, - add_file_name=False, - add_import_date=False, - custom_meta_data=None, camera_uuid=None, ) -> T.List[types.MetadataOrError]: - if add_file_name: - LOG.warning("The option --add_file_name is not needed any more since v0.10.0") - - if add_import_date: - LOG.warning("The option --add_import_date is not needed any more since v0.10.0") - - if custom_meta_data: - LOG.warning( - "The option --custom_meta_data is not needed any more since v0.10.0" - ) - for metadata in metadatas: if isinstance(metadata, types.ErrorMetadata): continue diff --git a/mapillary_tools/types.py b/mapillary_tools/types.py index d50a009fb..a7616a216 100644 --- a/mapillary_tools/types.py +++ b/mapillary_tools/types.py @@ -36,6 +36,7 @@ class FileType(enum.Enum): GOPRO = "gopro" IMAGE = "image" VIDEO = "video" + ZIP = "zip" @dataclasses.dataclass diff --git a/mapillary_tools/upload.py b/mapillary_tools/upload.py index 39a0cc8ea..b22b649a4 100644 --- a/mapillary_tools/upload.py +++ b/mapillary_tools/upload.py @@ -1,4 +1,3 @@ -import enum import json import logging import os @@ -17,7 +16,6 @@ config, constants, exceptions, - geo, history, ipc, telemetry, @@ -27,8 +25,8 @@ utils, VERSION, ) -from .camm import camm_builder, camm_parser -from .geotag import blackvue_parser, gpmf_parser, utils as video_utils +from .camm import camm_builder +from .geotag import gpmf_parser from .mp4 import simple_mp4_builder from .types import FileType @@ -53,12 +51,6 @@ class UploadHTTPError(Exception): pass -class DirectUploadFileType(enum.Enum): - RAW_BLACKVUE = "raw_blackvue" - RAW_CAMM = "raw_camm" - ZIP = "zip" - - def wrap_http_exception(ex: requests.HTTPError): req = ex.request resp = ex.response @@ -519,7 +511,6 @@ def _find_metadata_with_filename_existed_in( def upload( import_path: T.Union[Path, T.Sequence[Path]], - filetypes: T.Set[T.Union[FileType, DirectUploadFileType]], desc_path: T.Optional[str] = None, _metadatas_from_process: T.Optional[T.Sequence[types.MetadataOrError]] = None, user_name: T.Optional[str] = None, @@ -527,19 +518,6 @@ def upload( dry_run=False, skip_subfolders=False, ) -> None: - if ( - DirectUploadFileType.RAW_BLACKVUE in filetypes - and FileType.BLACKVUE in filetypes - ): - raise exceptions.MapillaryBadParameterError( - f"filetypes should contain either {DirectUploadFileType.RAW_BLACKVUE.value} or {FileType.BLACKVUE.value}, not both", - ) - - if DirectUploadFileType.RAW_CAMM in filetypes and FileType.CAMM in filetypes: - raise exceptions.MapillaryBadParameterError( - f"File types should contain either {DirectUploadFileType.RAW_CAMM.value} or {FileType.CAMM.value}, not both", - ) - import_paths: T.Sequence[Path] if isinstance(import_path, Path): import_paths = [import_path] @@ -558,14 +536,7 @@ def upload( f"Import file or directory not found: {path}" ) - if { - DirectUploadFileType.RAW_CAMM, - DirectUploadFileType.RAW_BLACKVUE, - DirectUploadFileType.ZIP, - }.issuperset(filetypes): - metadatas = None - else: - metadatas = _load_descs(_metadatas_from_process, desc_path, import_paths) + metadatas = _load_descs(_metadatas_from_process, desc_path, import_paths) user_items = fetch_user_items(user_name, organization_key) @@ -608,116 +579,79 @@ def upload( chunk_size=int(constants.UPLOAD_CHUNK_SIZE_MB * 1024 * 1024), ) - # if more than one filetypes speficied, check filename suffixes, - # i.e. files not ended with .jpg or .mp4 will be ignored - check_file_suffix = len(filetypes) > 1 - try: - if FileType.IMAGE in filetypes: - image_paths = utils.find_images( - import_paths, - skip_subfolders=skip_subfolders, - check_file_suffix=check_file_suffix, - ) - # find descs that match the image paths from the import paths - image_metadatas = [ - metadata - for metadata in (metadatas or []) - if isinstance(metadata, types.ImageMetadata) - ] - specified_image_metadatas = _find_metadata_with_filename_existed_in( - image_metadatas, image_paths + image_paths = utils.find_images(import_paths, skip_subfolders=skip_subfolders) + # find descs that match the image paths from the import paths + image_metadatas = [ + metadata + for metadata in (metadatas or []) + if isinstance(metadata, types.ImageMetadata) + ] + specified_image_metadatas = _find_metadata_with_filename_existed_in( + image_metadatas, image_paths + ) + if specified_image_metadatas: + try: + clusters = mly_uploader.upload_images( + specified_image_metadatas, + event_payload={"file_type": FileType.IMAGE.value}, + ) + except Exception as ex: + raise UploadError(ex) from ex + + if clusters: + LOG.debug("Uploaded to cluster: %s", clusters) + + video_paths = utils.find_videos(import_paths, skip_subfolders=skip_subfolders) + video_metadatas = [ + metadata + for metadata in (metadatas or []) + if isinstance(metadata, types.VideoMetadata) + ] + specified_video_metadatas = _find_metadata_with_filename_existed_in( + video_metadatas, video_paths + ) + for idx, video_metadata in enumerate(specified_video_metadatas): + video_metadata.update_md5sum() + assert isinstance(video_metadata.md5sum, str), "md5sum should be updated" + + # extract telemetry measurements from GoPro videos + telemetry_measurements: T.List[telemetry.TelemetryMeasurement] = [] + if MAPILLARY__EXPERIMENTAL_ENABLE_IMU == "YES": + if video_metadata.filetype is FileType.GOPRO: + with video_metadata.filename.open("rb") as fp: + telemetry_data = gpmf_parser.extract_telemetry_data(fp) + if telemetry_data: + telemetry_measurements.extend(telemetry_data.accl) + telemetry_measurements.extend(telemetry_data.gyro) + telemetry_measurements.extend(telemetry_data.magn) + telemetry_measurements.sort(key=lambda m: m.time) + + generator = camm_builder.camm_sample_generator2( + video_metadata, telemetry_measurements=telemetry_measurements ) - if specified_image_metadatas: + + with video_metadata.filename.open("rb") as src_fp: + camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator) + event_payload: uploader.Progress = { + "total_sequence_count": len(specified_video_metadatas), + "sequence_idx": idx, + "file_type": video_metadata.filetype.value, + "import_path": str(video_metadata.filename), + } try: - clusters = mly_uploader.upload_images( - specified_image_metadatas, - event_payload={"file_type": FileType.IMAGE.value}, + cluster_id = mly_uploader.upload_stream( + T.cast(T.BinaryIO, camm_fp), + upload_api_v4.ClusterFileType.CAMM, + video_metadata.md5sum, + event_payload=event_payload, ) except Exception as ex: raise UploadError(ex) from ex + LOG.debug("Uploaded to cluster: %s", cluster_id) - if clusters: - LOG.debug("Uploaded to cluster: %s", clusters) - - supported = CAMM_CONVERTABLES.intersection(filetypes) - if supported: - video_paths = utils.find_videos( - import_paths, - skip_subfolders=skip_subfolders, - check_file_suffix=check_file_suffix, - ) - video_metadatas = [ - metadata - for metadata in (metadatas or []) - if isinstance(metadata, types.VideoMetadata) - ] - specified_video_metadatas = _find_metadata_with_filename_existed_in( - video_metadatas, video_paths - ) - for idx, video_metadata in enumerate(specified_video_metadatas): - video_metadata.update_md5sum() - assert isinstance(video_metadata.md5sum, str), ( - "md5sum should be updated" - ) - - # extract telemetry measurements from GoPro videos - telemetry_measurements: T.List[telemetry.TelemetryMeasurement] = [] - if MAPILLARY__EXPERIMENTAL_ENABLE_IMU == "YES": - if video_metadata.filetype is FileType.GOPRO: - with video_metadata.filename.open("rb") as fp: - telemetry_data = gpmf_parser.extract_telemetry_data(fp) - if telemetry_data: - telemetry_measurements.extend(telemetry_data.accl) - telemetry_measurements.extend(telemetry_data.gyro) - telemetry_measurements.extend(telemetry_data.magn) - telemetry_measurements.sort(key=lambda m: m.time) - - generator = camm_builder.camm_sample_generator2( - video_metadata, telemetry_measurements=telemetry_measurements - ) - with video_metadata.filename.open("rb") as src_fp: - camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator) - event_payload: uploader.Progress = { - "total_sequence_count": len(specified_video_metadatas), - "sequence_idx": idx, - "file_type": video_metadata.filetype.value, - "import_path": str(video_metadata.filename), - } - try: - cluster_id = mly_uploader.upload_stream( - T.cast(T.BinaryIO, camm_fp), - upload_api_v4.ClusterFileType.CAMM, - video_metadata.md5sum, - event_payload=event_payload, - ) - except Exception as ex: - raise UploadError(ex) from ex - LOG.debug("Uploaded to cluster: %s", cluster_id) - - if DirectUploadFileType.RAW_BLACKVUE in filetypes: - video_paths = utils.find_videos( - import_paths, - skip_subfolders=skip_subfolders, - check_file_suffix=check_file_suffix, - ) - _upload_raw_blackvues_DEPRECATED(mly_uploader, video_paths) - - if DirectUploadFileType.RAW_CAMM in filetypes: - video_paths = utils.find_videos( - import_paths, - skip_subfolders=skip_subfolders, - check_file_suffix=check_file_suffix, - ) - _upload_raw_camm_DEPRECATED(mly_uploader, video_paths) - - if DirectUploadFileType.ZIP in filetypes: - zip_paths = utils.find_zipfiles( - import_paths, - skip_subfolders=skip_subfolders, - check_file_suffix=check_file_suffix, - ) - _upload_zipfiles(mly_uploader, zip_paths) + zip_paths = utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders) + _upload_zipfiles(mly_uploader, zip_paths) except UploadError as ex: inner_ex = ex.inner_ex @@ -756,111 +690,6 @@ def upload( LOG.info("Nothing uploaded. Bye.") -def _check_blackvue_DEPRECATED(video_path: Path) -> None: - # Skip in tests only because we don't have valid sample blackvue for tests - if os.getenv("MAPILLARY__DISABLE_BLACKVUE_CHECK") == "YES": - return - - points = blackvue_parser.parse_gps_points(video_path) - if not points: - raise exceptions.MapillaryGPXEmptyError("No GPS found in the BlackVue video") - - stationary = video_utils.is_video_stationary( - geo.get_max_distance_from_start([(p.lat, p.lon) for p in points]) - ) - if stationary: - raise exceptions.MapillaryStationaryVideoError("Stationary BlackVue video") - - -def _upload_raw_blackvues_DEPRECATED( - mly_uploader: uploader.Uploader, - video_paths: T.Sequence[Path], -) -> None: - for idx, video_path in enumerate(video_paths): - event_payload: uploader.Progress = { - "total_sequence_count": len(video_paths), - "sequence_idx": idx, - "file_type": DirectUploadFileType.RAW_BLACKVUE.value, - "import_path": str(video_path), - } - - try: - _check_blackvue_DEPRECATED(video_path) - except Exception as ex: - LOG.warning( - "Skipping %s %s due to: %s", - DirectUploadFileType.RAW_BLACKVUE.value.upper(), - video_path.name, - ex, - ) - continue - - with video_path.open("rb") as fp: - upload_md5sum = utils.md5sum_fp(fp).hexdigest() - try: - cluster_id = mly_uploader.upload_stream( - fp, - upload_api_v4.ClusterFileType.BLACKVUE, - upload_md5sum, - event_payload=event_payload, - ) - except Exception as ex: - raise UploadError(ex) from ex - LOG.debug("Uploaded to cluster: %s", cluster_id) - - -def _check_camm_DEPRECATED(video_path: Path) -> None: - # Skip in tests only because we don't have valid sample CAMM for tests - if os.getenv("MAPILLARY__DISABLE_CAMM_CHECK") == "YES": - return - - points = camm_parser.parse_gpx(video_path) - if not points: - raise exceptions.MapillaryGPXEmptyError("No GPS found in the CAMM video") - - stationary = video_utils.is_video_stationary( - geo.get_max_distance_from_start([(p.lat, p.lon) for p in points]) - ) - if stationary: - raise exceptions.MapillaryStationaryVideoError("Stationary CAMM video") - - -def _upload_raw_camm_DEPRECATED( - mly_uploader: uploader.Uploader, - video_paths: T.Sequence[Path], -) -> None: - for idx, video_path in enumerate(video_paths): - event_payload: uploader.Progress = { - "total_sequence_count": len(video_paths), - "sequence_idx": idx, - "file_type": DirectUploadFileType.RAW_CAMM.value, - "import_path": str(video_path), - } - try: - _check_camm_DEPRECATED(video_path) - except Exception as ex: - LOG.warning( - "Skipping %s %s due to: %s", - DirectUploadFileType.RAW_CAMM.value.upper(), - video_path.name, - ex, - ) - continue - try: - with video_path.open("rb") as fp: - upload_md5sum = utils.md5sum_fp(fp).hexdigest() - cluster_id = mly_uploader.upload_stream( - fp, - upload_api_v4.ClusterFileType.CAMM, - upload_md5sum, - event_payload=event_payload, - ) - except Exception as ex: - raise UploadError(ex) from ex - - LOG.debug("Uploaded to cluster: %s", cluster_id) - - def _upload_zipfiles( mly_uploader: uploader.Uploader, zip_paths: T.Sequence[Path], @@ -869,7 +698,7 @@ def _upload_zipfiles( event_payload: uploader.Progress = { "total_sequence_count": len(zip_paths), "sequence_idx": idx, - "file_type": DirectUploadFileType.ZIP.value, + "file_type": FileType.ZIP.value, "import_path": str(zip_path), } try: diff --git a/mapillary_tools/utils.py b/mapillary_tools/utils.py index d5e738a17..58b53bfc4 100644 --- a/mapillary_tools/utils.py +++ b/mapillary_tools/utils.py @@ -123,7 +123,6 @@ def deduplicate_paths(paths: T.Iterable[Path]) -> T.Generator[Path, None, None]: def find_images( import_paths: T.Sequence[Path], skip_subfolders: bool = False, - check_file_suffix: bool = False, ) -> T.List[Path]: image_paths: T.List[Path] = [] for path in import_paths: @@ -134,10 +133,7 @@ def find_images( if is_image_file(file) ) else: - if check_file_suffix: - if is_image_file(path): - image_paths.append(path) - else: + if is_image_file(path): image_paths.append(path) return list(deduplicate_paths(image_paths)) @@ -145,7 +141,6 @@ def find_images( def find_videos( import_paths: T.Sequence[Path], skip_subfolders: bool = False, - check_file_suffix: bool = False, ) -> T.List[Path]: video_paths: T.List[Path] = [] for path in import_paths: @@ -156,10 +151,7 @@ def find_videos( if is_video_file(file) ) else: - if check_file_suffix: - if is_video_file(path): - video_paths.append(path) - else: + if is_video_file(path): video_paths.append(path) return list(deduplicate_paths(video_paths)) @@ -167,7 +159,6 @@ def find_videos( def find_zipfiles( import_paths: T.Sequence[Path], skip_subfolders: bool = False, - check_file_suffix: bool = False, ) -> T.List[Path]: zip_paths: T.List[Path] = [] for path in import_paths: @@ -178,10 +169,7 @@ def find_zipfiles( if file.suffix.lower() in [".zip"] ) else: - if check_file_suffix: - if path.suffix.lower() in [".zip"]: - zip_paths.append(path) - else: + if path.suffix.lower() in [".zip"]: zip_paths.append(path) return list(deduplicate_paths(zip_paths)) @@ -199,5 +187,6 @@ def find_xml_files(import_paths: T.Sequence[Path]) -> T.List[Path]: if file.suffix.lower() in [".xml"] ) else: - xml_paths.append(path) + if path.suffix.lower() in [".xml"]: + xml_paths.append(path) return list(deduplicate_paths(xml_paths)) diff --git a/mapillary_tools/video_data_extraction/cli_options.py b/mapillary_tools/video_data_extraction/cli_options.py index 0e073729f..dbcd064fd 100644 --- a/mapillary_tools/video_data_extraction/cli_options.py +++ b/mapillary_tools/video_data_extraction/cli_options.py @@ -20,4 +20,3 @@ class CliOptions(T.TypedDict, total=False): num_processes: int device_make: T.Optional[str] device_model: T.Optional[str] - check_file_suffix: bool diff --git a/mapillary_tools/video_data_extraction/extract_video_data.py b/mapillary_tools/video_data_extraction/extract_video_data.py index 716d4657e..311286505 100644 --- a/mapillary_tools/video_data_extraction/extract_video_data.py +++ b/mapillary_tools/video_data_extraction/extract_video_data.py @@ -32,9 +32,7 @@ def __init__(self, options: CliOptions) -> None: def process(self) -> T.List[MetadataOrError]: paths = self.options["paths"] self._check_paths(paths) - video_files = utils.find_videos( - paths, check_file_suffix=self.options["check_file_suffix"] - ) + video_files = utils.find_videos(paths) self._check_sources_cardinality(video_files) num_processes = self.options["num_processes"] or None diff --git a/mapillary_tools/video_data_extraction/video_data_parser_factory.py b/mapillary_tools/video_data_extraction/video_data_parser_factory.py index 6239ea78b..2e5d1dfe9 100644 --- a/mapillary_tools/video_data_extraction/video_data_parser_factory.py +++ b/mapillary_tools/video_data_extraction/video_data_parser_factory.py @@ -1,14 +1,14 @@ import typing as T from pathlib import Path -from .extractors.exiftool_runtime_parser import ExiftoolRuntimeParser - from .cli_options import CliOptions from .extractors.base_parser import BaseParser from .extractors.blackvue_parser import BlackVueParser from .extractors.camm_parser import CammParser + +from .extractors.exiftool_runtime_parser import ExiftoolRuntimeParser from .extractors.exiftool_xml_parser import ExiftoolXmlParser from .extractors.generic_video_parser import GenericVideoParser from .extractors.gopro_parser import GoProParser diff --git a/tests/integration/fixtures.py b/tests/integration/fixtures.py index c84f4d570..52b8d9c36 100644 --- a/tests/integration/fixtures.py +++ b/tests/integration/fixtures.py @@ -53,8 +53,6 @@ def setup_data(tmpdir: py.path.local): def setup_upload(tmpdir: py.path.local): upload_dir = tmpdir.mkdir("mapillary_public_uploads") os.environ["MAPILLARY_UPLOAD_PATH"] = str(upload_dir) - os.environ["MAPILLARY__DISABLE_BLACKVUE_CHECK"] = "YES" - os.environ["MAPILLARY__DISABLE_CAMM_CHECK"] = "YES" os.environ["MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN"] = "YES" history_path = tmpdir.join("history") os.environ["MAPILLARY_UPLOAD_HISTORY_PATH"] = str(history_path) @@ -62,7 +60,6 @@ def setup_upload(tmpdir: py.path.local): if tmpdir.check(): tmpdir.remove(ignore_errors=True) del os.environ["MAPILLARY_UPLOAD_PATH"] - del os.environ["MAPILLARY__DISABLE_BLACKVUE_CHECK"] del os.environ["MAPILLARY_UPLOAD_HISTORY_PATH"] del os.environ["MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN"] diff --git a/tests/integration/test_history.py b/tests/integration/test_history.py index 565c0e5a5..656bc626f 100644 --- a/tests/integration/test_history.py +++ b/tests/integration/test_history.py @@ -75,25 +75,27 @@ def test_upload_images( @pytest.mark.usefixtures("setup_config") -def test_upload_blackvue( +def test_upload_gopro( setup_data: py.path.local, setup_upload: py.path.local, ): assert len(setup_upload.listdir()) == 0 - video_dir = setup_data.join("videos") + video_dir = setup_data.join("gopro_data") x = subprocess.run( - f"{EXECUTABLE} upload_blackvue {UPLOAD_FLAGS} {str(video_dir)}", + f"{EXECUTABLE} process_and_upload --skip_process_errors {UPLOAD_FLAGS} {str(video_dir)}", shell=True, ) assert x.returncode == 0, x.stderr - assert len(setup_upload.listdir()) == 1, "should be uploaded for the first time" + assert len(setup_upload.listdir()) == 1, ( + f"should be uploaded for the first time but got {setup_upload.listdir()}" + ) for upload in setup_upload.listdir(): upload.remove() assert len(setup_upload.listdir()) == 0 x = subprocess.run( - f"{EXECUTABLE} upload_blackvue {UPLOAD_FLAGS} {str(video_dir)}", + f"{EXECUTABLE} process_and_upload --skip_process_errors {UPLOAD_FLAGS} {str(video_dir)}", shell=True, ) assert x.returncode == 0, x.stderr diff --git a/tests/integration/test_process.py b/tests/integration/test_process.py index 68083571a..2a00d5ba8 100644 --- a/tests/integration/test_process.py +++ b/tests/integration/test_process.py @@ -21,7 +21,7 @@ ) -PROCESS_FLAGS = "--add_import_date" +PROCESS_FLAGS = "" _DEFAULT_EXPECTED_DESCS = { "DSC00001.JPG": { @@ -311,8 +311,6 @@ def test_zip(tmpdir: py.path.local, setup_data: py.path.local): @pytest.mark.usefixtures("setup_config") def test_process_boolean_options(setup_data: py.path.local): boolean_options = [ - "--add_file_name", - "--add_import_date", "--interpolate_directions", "--overwrite_EXIF_direction_tag", "--overwrite_EXIF_gps_tag", diff --git a/tests/integration/test_process_and_upload.py b/tests/integration/test_process_and_upload.py index 4325a6c6f..5d74b3cf7 100644 --- a/tests/integration/test_process_and_upload.py +++ b/tests/integration/test_process_and_upload.py @@ -16,7 +16,7 @@ validate_and_extract_zip, ) -PROCESS_FLAGS = "--add_import_date" +PROCESS_FLAGS = "" UPLOAD_FLAGS = f"--dry_run --user_name={USERNAME}" @@ -150,10 +150,7 @@ def _validate_output(upload_dir: py.path.local, expected): @pytest.mark.usefixtures("setup_config") -def test_process_and_upload( - setup_data: py.path.local, - setup_upload: py.path.local, -): +def test_process_and_upload(setup_data: py.path.local, setup_upload: py.path.local): input_paths = [ setup_data.join("videos"), setup_data.join("gopro_data"), diff --git a/tests/integration/test_upload.py b/tests/integration/test_upload.py index 19128729c..1d0131a01 100644 --- a/tests/integration/test_upload.py +++ b/tests/integration/test_upload.py @@ -16,7 +16,7 @@ ) -PROCESS_FLAGS = "--add_import_date" +PROCESS_FLAGS = "" UPLOAD_FLAGS = f"--dry_run --user_name={USERNAME}" @@ -31,97 +31,6 @@ def file_md5sum(path) -> str: return md5.hexdigest() -@pytest.mark.usefixtures("setup_config") -def test_upload_multiple_mp4s_DEPRECATED( - setup_data: py.path.local, - setup_upload: py.path.local, -): - video_path = setup_data.join("videos").join("sample-5s.mp4") - x = subprocess.run( - f"{EXECUTABLE} upload_blackvue {UPLOAD_FLAGS} {video_path} {video_path}", - shell=True, - ) - - assert 1 == len(setup_upload.listdir()) - assert {"mly_tools_8cd0e9af15f4baaafe9dfe98ace8b886.mp4"} == { - os.path.basename(f) for f in setup_upload.listdir() - } - md5sum = file_md5sum(video_path) - assert {md5sum} == {file_md5sum(f) for f in setup_upload.listdir()} - - -@pytest.mark.usefixtures("setup_config") -def test_upload_blackvue( - tmpdir: py.path.local, - setup_data: py.path.local, - setup_upload: py.path.local, -): - another_path = tmpdir.join("another_sub") - - video_path2 = another_path.join("sub1 folder").join("sub2 folder").join("hello.mp4") - video_path2.write_text("hello", encoding="utf-8", ensure=True) - - video_path_invalid_ext = ( - another_path.join("sub1 folder").join("sub2 folder").join("hello.mp45") - ) - video_path_invalid_ext.write_text("hello2", encoding="utf-8", ensure=True) - - hidden_video_path3 = another_path.join(".subfolder").join("hello.mp4") - hidden_video_path3.write_text("world", encoding="utf-8", ensure=True) - - video_path_hello2 = tmpdir.join("sub1 folder").join("sub2 folder").join("hello.mp4") - video_path_hello2.write_text("hello2", encoding="utf-8", ensure=True) - - video_dir = setup_data.join("videos") - x = subprocess.run( - f'{EXECUTABLE} upload_blackvue {UPLOAD_FLAGS} {str(video_dir)} {str(another_path)} "{str(video_path2)}" "{str(video_path_hello2)}"', - shell=True, - ) - assert x.returncode == 0, x.stderr - - assert { - "mly_tools_8cd0e9af15f4baaafe9dfe98ace8b886.mp4", - f"mly_tools_{file_md5sum(str(video_path2))}.mp4", - f"mly_tools_{file_md5sum(str(video_path_hello2))}.mp4", - } == {os.path.basename(f) for f in setup_upload.listdir()} - - -@pytest.mark.usefixtures("setup_config") -def test_upload_camm( - tmpdir: py.path.local, - setup_data: py.path.local, - setup_upload: py.path.local, -): - another_path = tmpdir.join("another_sub") - - video_path2 = another_path.join("sub1 folder").join("sub2 folder").join("hello.mp4") - video_path2.write_text("hello", encoding="utf-8", ensure=True) - - video_path_invalid_ext = ( - another_path.join("sub1 folder").join("sub2 folder").join("hello.mp45") - ) - video_path_invalid_ext.write_text("hello2", encoding="utf-8", ensure=True) - - hidden_video_path3 = another_path.join(".subfolder").join("hello.mp4") - hidden_video_path3.write_text("world", encoding="utf-8", ensure=True) - - video_path_hello2 = tmpdir.join("sub1 folder").join("sub2 folder").join("hello.mp4") - video_path_hello2.write_text("hello2", encoding="utf-8", ensure=True) - - video_dir = setup_data.join("videos") - x = subprocess.run( - f'{EXECUTABLE} upload_camm {UPLOAD_FLAGS} {str(video_dir)} {str(another_path)} "{str(video_path2)}" "{str(video_path_hello2)}"', - shell=True, - ) - assert x.returncode == 0, x.stderr - - assert { - "mly_tools_8cd0e9af15f4baaafe9dfe98ace8b886.mp4", - f"mly_tools_{file_md5sum(str(video_path2))}.mp4", - f"mly_tools_{file_md5sum(str(video_path_hello2))}.mp4", - } == {os.path.basename(f) for f in setup_upload.listdir()} - - @pytest.mark.usefixtures("setup_config") def test_upload_image_dir( setup_data: py.path.local, @@ -133,7 +42,7 @@ def test_upload_image_dir( ) assert x.returncode == 0, x.stderr x = subprocess.run( - f"{EXECUTABLE} upload {UPLOAD_FLAGS} --file_types=image {setup_data}", + f"{EXECUTABLE} process_and_upload {UPLOAD_FLAGS} --file_types=image {setup_data}", shell=True, ) for file in setup_upload.listdir(): @@ -157,7 +66,7 @@ def test_upload_image_dir_twice( # first upload x = subprocess.run( - f"{EXECUTABLE} upload {UPLOAD_FLAGS} --file_types=image {setup_data}", + f"{EXECUTABLE} process_and_upload {UPLOAD_FLAGS} --file_types=image {setup_data}", shell=True, ) assert x.returncode == 0, x.stderr @@ -167,7 +76,7 @@ def test_upload_image_dir_twice( # expect the second upload to not produce new uploads x = subprocess.run( - f"{EXECUTABLE} upload {UPLOAD_FLAGS} --desc_path={desc_path} --file_types=image {setup_data} {setup_data} {setup_data}/images/DSC00001.JPG", + f"{EXECUTABLE} process_and_upload {UPLOAD_FLAGS} --desc_path={desc_path} --file_types=image {setup_data} {setup_data} {setup_data}/images/DSC00001.JPG", shell=True, ) assert x.returncode == 0, x.stderr @@ -209,30 +118,3 @@ def test_upload_wrong_descs( shell=True, ) assert x.returncode == 4, x.stderr - - -@pytest.mark.usefixtures("setup_config") -def test_upload_zip( - tmpdir: py.path.local, - setup_data: py.path.local, - setup_upload: py.path.local, -): - zip_dir = tmpdir.mkdir("zip_dir") - x = subprocess.run( - f"{EXECUTABLE} process --file_types=image {PROCESS_FLAGS} {setup_data}", - shell=True, - ) - assert x.returncode == 0, x.stderr - x = subprocess.run( - f"{EXECUTABLE} zip {setup_data} {zip_dir}", - shell=True, - ) - assert x.returncode == 0, x.stderr - for zfile in zip_dir.listdir(): - x = subprocess.run( - f"{EXECUTABLE} upload_zip {UPLOAD_FLAGS} {zfile} {zfile}", - shell=True, - ) - assert x.returncode == 0, x.stderr - for file in setup_upload.listdir(): - validate_and_extract_zip(str(file)) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index c75d9722b..0f735e409 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -80,7 +80,7 @@ def test_filter_all(tmpdir: py.path.local): tmpdir.join("foo").mkdir(".git") # TODO: life too short to test for windows if not sys.platform.startswith("win"): - assert {"foo/world.TIFF", "foo/hello.jpg", "foo/.foo", "foo/hello.jpe"} == set( + assert {"foo/world.TIFF", "foo/hello.jpg", "foo/hello.jpe"} == set( str(p.relative_to(tmpdir)) for p in utils.find_images( [ @@ -91,7 +91,7 @@ def test_filter_all(tmpdir: py.path.local): ] ) ) - assert {"foo/world.zip", "foo/.foo", "foo/world.ZIP"} == set( + assert {"foo/world.zip", "foo/world.ZIP"} == set( str(p.relative_to(tmpdir)) for p in utils.find_zipfiles( [ @@ -116,7 +116,7 @@ def test_filter_all(tmpdir: py.path.local): ) # some platform filenames are case sensitive? assert ( - {"foo/world.MP4", "foo/.foo", "foo/world.ts"} == actual - or {"foo/world.mp4", "foo/world.MP4", "foo/.foo", "foo/world.ts"} == actual - or {"foo/world.mp4", "foo/.foo", "foo/world.ts"} == actual + {"foo/world.MP4", "foo/world.ts"} == actual + or {"foo/world.mp4", "foo/world.MP4", "foo/world.ts"} == actual + or {"foo/world.mp4", "foo/world.ts"} == actual )