diff --git a/setup.cfg b/setup.cfg index 5e40900..2bc7ce7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ +[aliases] +test=pytest + [wheel] universal = 1 diff --git a/setup.py b/setup.py index 5138cf3..fd281ac 100644 --- a/setup.py +++ b/setup.py @@ -27,8 +27,12 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6' + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7' ], packages=['sigmf'], + install_requires=['six', 'numpy'], + setup_requires=['pytest-runner'], + tests_require=['pytest>3'], zip_safe=False ) diff --git a/sigmf/__init__.py b/sigmf/__init__.py index 7e23785..49622cb 100644 --- a/sigmf/__init__.py +++ b/sigmf/__init__.py @@ -7,8 +7,8 @@ # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, @@ -17,10 +17,5 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -""" -Cyberspectrum annotation format -""" -from .sigmffile import SigMFFile - -__version__ = "1.0.0" +__version__ = "0.0.1" diff --git a/sigmf/archive.py b/sigmf/archive.py new file mode 100644 index 0000000..bab6bbd --- /dev/null +++ b/sigmf/archive.py @@ -0,0 +1,166 @@ +# Copyright 2017 GNU Radio Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +"""Create and extract SigMF archives.""" + +import os +import shutil +import tarfile +import tempfile + +from . import error + + +SIGMF_ARCHIVE_EXT = ".sigmf" +SIGMF_METADATA_EXT = ".sigmf-meta" +SIGMF_DATASET_EXT = ".sigmf-data" + + +class SigMFArchive(object): + """Archive a SigMFFile. + + A `.sigmf` file must include both valid metadata and data. If metadata + is not valid, raise `SigMFValidationError`. If `self.data_file` is not + set or the requested output file is not writable, raise `SigMFFileError`. + + Parameters: + + sigmffile -- A SigMFFile object with valid metadata and data_file + + name -- path to archive file to create. If file exists, overwrite. + If `name` doesn't end in .sigmf, it will be appended. + For example: if `name` == "/tmp/archive1", then the + following archive will be created: + /tmp/archive1.sigmf + - archive1/ + - archive1.sigmf-meta + - archive1.sigmf-data + + fileobj -- If `fileobj` is specified, it is used as an alternative to + a file object opened in binary mode for `name`. It is + supposed to be at position 0. `name` is not required, but + if specified will be used to determine the directory and + file names within the archive. `fileobj` won't be closed. + For example: if `name` == "archive1" and fileobj is given, + a tar archive will be written to fileobj with the + following structure: + - archive1/ + - archive1.sigmf-meta + - archive1.sigmf-data + + """ + def __init__(self, sigmffile, name=None, fileobj=None): + self.sigmffile = sigmffile + self.name = name + self.fileobj = fileobj + + self._check_input() + + archive_name = self._get_archive_name() + sigmf_fileobj = self._get_output_fileobj() + sigmf_archive = tarfile.TarFile(mode="w", fileobj=sigmf_fileobj) + tmpdir = tempfile.mkdtemp() + sigmf_md_filename = archive_name + SIGMF_METADATA_EXT + sigmf_md_path = os.path.join(tmpdir, sigmf_md_filename) + sigmf_data_filename = archive_name + SIGMF_DATASET_EXT + sigmf_data_path = os.path.join(tmpdir, sigmf_data_filename) + + with open(sigmf_md_path, "w") as mdfile: + self.sigmffile.dump(mdfile, pretty=True) + + shutil.copy(self.sigmffile.data_file, sigmf_data_path) + + def chmod(tarinfo): + if tarinfo.isdir(): + tarinfo.mode = 0o755 # dwrxw-rw-r + else: + tarinfo.mode = 0o644 # -wr-r--r-- + return tarinfo + + sigmf_archive.add(tmpdir, arcname=archive_name, filter=chmod) + sigmf_archive.close() + if not fileobj: + sigmf_fileobj.close() + + shutil.rmtree(tmpdir) + + self.path = sigmf_archive.name + + def _check_input(self): + self._ensure_name_has_correct_extension() + self._ensure_data_file_set() + self._validate_sigmffile_metadata() + + def _ensure_name_has_correct_extension(self): + name = self.name + if name is None: + return + + has_extension = "." in name + has_correct_extension = name.endswith(SIGMF_ARCHIVE_EXT) + if has_extension and not has_correct_extension: + apparent_ext = os.path.splitext(name)[-1] + err = "extension {} != {}".format(apparent_ext, SIGMF_ARCHIVE_EXT) + raise error.SigMFFileError(err) + + self.name = name if has_correct_extension else name + SIGMF_ARCHIVE_EXT + + def _ensure_data_file_set(self): + if not self.sigmffile.data_file: + err = "no data file - use `set_data_file`" + raise error.SigMFFileError(err) + + def _validate_sigmffile_metadata(self): + valid_md = self.sigmffile.validate() + if not valid_md: + err = "invalid metadata - {!s}" + raise error.SigMFValidationError(err.format(valid_md)) + + def _get_archive_name(self): + if self.fileobj and not self.name: + pathname = self.fileobj.name + else: + pathname = self.name + + filename = os.path.split(pathname)[-1] + archive_name, archive_ext = os.path.splitext(filename) + return archive_name + + def _get_output_fileobj(self): + try: + fileobj = self._get_open_fileobj() + except: + if self.fileobj: + e = "fileobj {!r} is not byte-writable".format(self.fileobj) + else: + e = "can't open {!r} for writing".format(self.name) + + raise error.SigMFFileError(e) + + return fileobj + + def _get_open_fileobj(self): + if self.fileobj: + fileobj = self.fileobj + fileobj.write(bytes()) # force exception if not byte-writable + else: + fileobj = open(self.name, "wb") + + return fileobj diff --git a/sigmf/error.py b/sigmf/error.py new file mode 100644 index 0000000..050733a --- /dev/null +++ b/sigmf/error.py @@ -0,0 +1,16 @@ +"""Defines SigMF exception classes.""" + + +class SigMFError(Exception): + """ SigMF base exception.""" + pass + + +class SigMFValidationError(SigMFError): + """Exceptions related to validating SigMF metadata.""" + pass + + +class SigMFFileError(SigMFError): + """Exceptions related to reading or writing SigMF archives.""" + pass diff --git a/sigmf/schema.json b/sigmf/schema.json index 19cf7f2..5d43214 100644 --- a/sigmf/schema.json +++ b/sigmf/schema.json @@ -1,106 +1,106 @@ { - "global": { - "required": true, - "type": "dict", - "keys": { - "core:datatype": { - "type": "string", - "required": true, - "help": "Sample data format" - }, - "core:offset": { - "type": "uint", - "required": false, - "help": "Index offset of the first sample. Defaults to 0" - }, - "core:description": { - "type": "string", - "required": false, - "help": "Textual description of the capture." - }, - "core:author": { - "type": "string", - "required": false, - "help": "Name and optionally email address of the author" - }, - "core:license": { - "type": "string", - "required": false, - "help": "Sample data license" - }, - "core:date": { - "type": "string", - "required": false, - "pattern": "", - "help": "ISO 8601-formatted date (e.g., 2017-02-01T15:05:03+00:00)" - }, - "core:sha512": { - "type": "string", - "required": false, - "help": "SHA512 hash of the corresponding sample data file" - }, - "core:version": { - "type": "string", - "required": true, - "default": "1.0.0", - "help": "Version of the data format header foo" - }, - "core:hw": { - "type": "string", - "required": false, - "help": "Information about the hardware used (measurement setup, antennas, etc.)" - } - } - }, + "global": { + "required": true, + "type": "dict", + "keys": { + "core:datatype": { + "type": "string", + "required": true, + "help": "Sample data format" + }, + "core:offset": { + "type": "uint", + "required": false, + "help": "Index offset of the first sample. Defaults to 0" + }, + "core:description": { + "type": "string", + "required": false, + "help": "Textual description of the capture." + }, + "core:author": { + "type": "string", + "required": false, + "help": "Name and optionally email address of the author" + }, + "core:license": { + "type": "string", + "required": false, + "help": "Sample data license" + }, + "core:date": { + "type": "string", + "required": false, + "pattern": "", + "help": "ISO 8601-formatted date (e.g., 2017-02-01T15:05:03+00:00)" + }, + "core:sha512": { + "type": "string", + "required": false, + "help": "SHA512 hash of the corresponding sample data file" + }, + "core:version": { + "type": "string", + "required": true, + "default": null, + "help": "Version of the SigMF specification" + }, + "core:hw": { + "type": "string", + "required": false, + "help": "Information about the hardware used (measurement setup, antennas, etc.)" + } + } + }, - "capture": { - "required": true, - "type": "dict_list", - "sort": "core:sample_start", - "keys": { - "core:sample_start": { - "type": "uint", - "required": true, - "help": "Index of first sample of this chunk" - }, - "core:frequency": { - "type": "double", - "required": false, - "help": "Center frequency of signal (Hz)" - }, - "core:sampling_rate": { - "type": "double", - "required": false, - "help": "Sampling rate of signal (Sps)" - }, - "core:time": { - "type": "string", - "required": false, - "help": "Start time of chunk" - } - } - }, + "captures": { + "required": true, + "type": "dict_list", + "sort": "core:sample_start", + "keys": { + "core:sample_start": { + "type": "uint", + "required": true, + "help": "Index of first sample of this chunk" + }, + "core:frequency": { + "type": "double", + "required": false, + "help": "Center frequency of signal (Hz)" + }, + "core:sampling_rate": { + "type": "double", + "required": false, + "help": "Sampling rate of signal (Sps)" + }, + "core:time": { + "type": "string", + "required": false, + "help": "Start time of chunk" + } + } + }, - "annotations": { - "required": true, - "type": "dict_list", - "sort": "core:sample_start", - "keys": { - "core:sample_start": { - "type": "uint", - "required": true, - "help": "Index of first sample of this chunk" - }, - "core:sample_count": { - "type": "uint", - "required": true, - "help": "The number of samples described by this segment" - }, - "core:comment": { - "type": "string", - "required": false, - "help": "Comment" - } - } - } + "annotations": { + "required": true, + "type": "dict_list", + "sort": "core:sample_start", + "keys": { + "core:sample_start": { + "type": "uint", + "required": true, + "help": "Index of first sample of this chunk" + }, + "core:sample_count": { + "type": "uint", + "required": true, + "help": "The number of samples described by this segment" + }, + "core:comment": { + "type": "string", + "required": false, + "help": "Comment" + } + } + } } diff --git a/sigmf/schema.py b/sigmf/schema.py index c095dff..0b17f34 100644 --- a/sigmf/schema.py +++ b/sigmf/schema.py @@ -17,18 +17,14 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -""" -Schema default foo. -""" import os import json -from sigmf import utils + +from . import utils + def get_schema(version=None): - """ - Return a schema based on the version - """ schema_file = os.path.join( utils.get_schema_path(os.path.dirname(utils.__file__)), 'schema.json' diff --git a/sigmf/sigmffile.py b/sigmf/sigmffile.py index dc0467e..2dfcc45 100644 --- a/sigmf/sigmffile.py +++ b/sigmf/sigmffile.py @@ -7,8 +7,8 @@ # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, @@ -17,55 +17,37 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -""" -SigMF File Representation Object -""" +import codecs import json +import tarfile +import tempfile +from os import path from six import iteritems -from sigmf.utils import dict_merge, insert_sorted_dict_list -from sigmf import validate -from sigmf import schema -def get_default_metadata(schema): - """ - Return a valid annotation object based on defaults. - """ - def get_default_dict(keys_dict): - " Return a dict with all default values from keys_dict " - return { - key: desc.get("default") - for key, desc in iteritems(keys_dict) - if "default" in desc - } - def default_category_data(cat_type, defaults): - " Return a valid data type for a category " - return { - 'dict': lambda x: x, - 'dict_list': lambda x: [x], - }[cat_type](defaults) - return { - category: default_category_data(desc["type"], get_default_dict(desc["keys"])) - for category, desc in iteritems(schema) - } +from . import __version__, schema, sigmf_hash, validate +from .archive import SigMFArchive, SIGMF_DATASET_EXT, SIGMF_METADATA_EXT +from .utils import dict_merge, insert_sorted_dict_list + class SigMFFile(object): - """ - API to manipulate annotation files. + """API to manipulate SigMF files. Parameters: - metadata -- Metadata. Either a string, or a dictionary. - data_file -- Path to the corresponding data file. - global_info -- Dictionary containing global header info. + + metadata -- Metadata. Either a string, or a dictionary. + data_file -- Path to the corresponding data file. + global_info -- Dictionary containing global header info. + """ START_INDEX_KEY = "core:sample_start" - LENGTH_INDEX_KEY = "core:sample_length" + LENGTH_INDEX_KEY = "core:sample_count" START_OFFSET_KEY = "core:offset" HASH_KEY = "core:sha512" VERSION_KEY = "core:version" GLOBAL_KEY = "global" - CAPTURE_KEY = "capture" - ANNOTATION_KEY = "annotation" + CAPTURE_KEY = "captures" + ANNOTATION_KEY = "annotations" def __init__( self, @@ -77,6 +59,8 @@ def __init__( self.schema = None if metadata is None: self._metadata = get_default_metadata(self.get_schema()) + if not self._metadata[self.GLOBAL_KEY][self.VERSION_KEY]: + self._metadata[self.GLOBAL_KEY][self.VERSION_KEY] = __version__ elif isinstance(metadata, dict): self._metadata = metadata else: @@ -84,6 +68,8 @@ def __init__( if global_info is not None: self.set_global_info(global_info) self.data_file = data_file + if self.data_file: + self.calculate_hash() def _get_start_offset(self): """ @@ -97,10 +83,9 @@ def _validate_dict_in_section(self, entries, section_key): Throws if not. """ schema_section = self.get_schema()[section_key] - print(schema_section) for k, v in iteritems(entries): validate.validate_key_throw( - v, schema_section.get(key, {}), section, k + v, schema_section.get(k, {}), schema_section, k ) def get_schema(self): @@ -110,7 +95,6 @@ def get_schema(self): current_metadata_version = self.get_global_info().get(self.VERSION_KEY) if self.version != current_metadata_version or self.schema is None: self.version = current_metadata_version - from sigmf import schema self.schema = schema.get_schema(self.version) assert isinstance(self.schema, dict) return self.schema @@ -138,7 +122,6 @@ def set_global_field(self, key, value): Will throw a ValueError if the key/value pair is invalid. """ schema_section = self.get_schema()[self.GLOBAL_KEY].get('keys', {}) - print( schema_section.get(key, {})) validate.validate_key_throw( value, schema_section.get(key, {}), @@ -186,12 +169,13 @@ def get_capture_info(self, index): cap_info = dict_merge(cap_info, capture) return cap_info - def add_annotation(self, start_index, length, metadata): + def add_annotation(self, start_index, length, metadata=None): """ Insert annotation """ assert start_index >= self._get_start_offset() assert length > 1 + metadata = metadata or {} metadata[self.START_INDEX_KEY] = start_index metadata[self.LENGTH_INDEX_KEY] = length self._validate_dict_in_section(metadata, self.ANNOTATION_KEY) @@ -217,10 +201,16 @@ def calculate_hash(self): Calculates the hash of the data file and adds it to the global section. Also returns a string representation of the hash. """ - from sigmf import sigmf_hash the_hash = sigmf_hash.calculate_sha512(self.data_file) return self.set_global_field(self.HASH_KEY, the_hash) + def set_data_file(self, data_file): + """ + Set the datafile path and recalculate the hash. Return the hash string. + """ + self.data_file = data_file + return self.calculate_hash() + def validate(self): """ Return True if the metadata is valid. @@ -233,7 +223,7 @@ def validate(self): def dump(self, filep, pretty=False): """ - Write out the file. + Write metadata to a file. Parameters: filep -- File pointer or something that json.dump() can handle @@ -258,3 +248,69 @@ def dumps(self, pretty=False): indent=4 if pretty else None, separators=(',', ': ') if pretty else None, ) + + def archive(self, name=None, fileobj=None): + """Dump contents to SigMF archive format. + + `name` and `fileobj` are passed to SigMFArchive and are defined there. + + """ + archive = SigMFArchive(self, name, fileobj) + return archive.path + + +def get_default_metadata(schema): + """Return the minimal metadata that will pass the validator.""" + def get_default_dict(keys_dict): + " Return a dict with all default values from keys_dict " + return { + key: desc.get("default") + for key, desc in iteritems(keys_dict) + if "default" in desc + } + + def default_category_data(cat_type, defaults): + " Return a valid data type for a category " + return { + 'dict': lambda x: x, + 'dict_list': lambda x: [x] if x else [], + }[cat_type](defaults) + + return { + category: default_category_data(desc["type"], get_default_dict(desc["keys"])) + for category, desc in iteritems(schema) + } + + +def fromarchive(archive_path, dir=None): + """Extract an archive and return a SigMFFile. + + If `dir` is given, extract the archive to that directory. Otherwise, + the archive will be extracted to a temporary directory. For example, + `dir` == "." will extract the archive into the current working + directory. + + """ + if not dir: + dir = tempfile.mkdtemp() + + archive = tarfile.open(archive_path) + members = archive.getmembers() + + try: + archive.extractall(path=dir) + + data_file = None + metadata = None + + for member in members: + if member.name.endswith(SIGMF_DATASET_EXT): + data_file = path.join(dir, member.name) + elif member.name.endswith(SIGMF_METADATA_EXT): + bytestream_reader = codecs.getreader("utf-8") # bytes -> str + mdfile_reader = bytestream_reader(archive.extractfile(member)) + metadata = json.load(mdfile_reader) + finally: + archive.close() + + return SigMFFile(metadata=metadata, data_file=data_file) diff --git a/sigmf/utils.py b/sigmf/utils.py index a862a79..f808a9d 100644 --- a/sigmf/utils.py +++ b/sigmf/utils.py @@ -22,8 +22,22 @@ """ from copy import deepcopy +from datetime import datetime + from six import iteritems + +SIGMF_DATETIME_ISO8601_FMT = "%Y-%m-%dT%H:%M:%S.%fZ" + + +def get_sigmf_iso8601_datetime_now(): + return datetime.isoformat(datetime.utcnow()) + 'Z' + + +def parse_iso8601_datetime(d): + return datetime.strptime(d, SIGMF_DATETIME_ISO8601_FMT) + + def dict_merge(a, b): """ Recursively merge b into a. b[k] will overwrite a[k] if it exists. @@ -44,6 +58,8 @@ def insert_sorted_dict_list(dict_list, new_entry, key): Returns the new list, which is still sorted. """ for index, entry in enumerate(dict_list): + if not entry: + continue if entry[key] == new_entry[key]: dict_list[index] = dict_merge(entry, new_entry) return dict_list @@ -57,4 +73,3 @@ def get_schema_path(module_path): """ """ return module_path - diff --git a/sigmf/validate.py b/sigmf/validate.py index 1f1fe08..ca73469 100644 --- a/sigmf/validate.py +++ b/sigmf/validate.py @@ -17,11 +17,14 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -" SigMF Validation routines " + +"""SigMF Validation routines""" from __future__ import print_function + import json + class ValidationResult(object): " Amends a validation result (True, False) with an error string. " def __init__(self, value=False, error=None): @@ -152,11 +155,6 @@ def validate_section(data_section, ref_section, section): }[ref_section["type"]](data_section, ref_section, section) def validate(data, ref=None): - """ - docstring for validate - - data, ref: dicts - """ if ref is None: from sigmf import schema ref = schema.get_schema() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a3365de --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,45 @@ +# Copyright 2017 GNU Radio Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import tempfile + +import pytest + +from sigmf.sigmffile import SigMFFile + +from .testdata import TEST_FLOAT32_DATA, TEST_METADATA + + +@pytest.yield_fixture +def test_data_file(): + with tempfile.NamedTemporaryFile() as t: + TEST_FLOAT32_DATA.tofile(t.name) + yield t + + +@pytest.fixture +def test_sigmffile(test_data_file): + f = SigMFFile() + f.set_global_field("core:datatype", "f32") + f.add_annotation(start_index=0, length=len(TEST_FLOAT32_DATA)) + f.add_capture(start_index=0) + f.set_data_file(test_data_file.name) + assert f._metadata == TEST_METADATA + return f diff --git a/tests/test_archive.py b/tests/test_archive.py new file mode 100644 index 0000000..34b8542 --- /dev/null +++ b/tests/test_archive.py @@ -0,0 +1,138 @@ +import codecs +import json +import os +import tarfile +import tempfile +from os import path + +import numpy as np +import pytest + +from sigmf import error +from sigmf.archive import SIGMF_DATASET_EXT, SIGMF_METADATA_EXT + +from .testdata import TEST_FLOAT32_DATA, TEST_METADATA + + +def create_test_archive(test_sigmffile, tmpfile): + sigmf_archive = test_sigmffile.archive(fileobj=tmpfile) + sigmf_tarfile = tarfile.open(sigmf_archive, mode="r") + return sigmf_tarfile + + +def test_without_data_file_throws_fileerror(test_sigmffile): + test_sigmffile.data_file = None + with tempfile.NamedTemporaryFile() as t: + with pytest.raises(error.SigMFFileError): + test_sigmffile.archive(name=t.name) + + +def test_invalid_md_throws_validationerror(test_sigmffile): + del test_sigmffile._metadata["global"]["core:datatype"] # required field + with tempfile.NamedTemporaryFile() as t: + with pytest.raises(error.SigMFValidationError): + test_sigmffile.archive(name=t.name) + + +def test_name_wrong_extension_throws_fileerror(test_sigmffile): + with tempfile.NamedTemporaryFile() as t: + with pytest.raises(error.SigMFFileError): + test_sigmffile.archive(name=t.name + ".zip") + + +def test_fileobj_extension_ignored(test_sigmffile): + with tempfile.NamedTemporaryFile(suffix=".tar") as t: + test_sigmffile.archive(fileobj=t) + + +def test_name_used_in_fileobj(test_sigmffile): + with tempfile.NamedTemporaryFile() as t: + sigmf_archive = test_sigmffile.archive(name="testarchive", fileobj=t) + sigmf_tarfile = tarfile.open(sigmf_archive, mode="r") + basedir, file1, file2 = sigmf_tarfile.getmembers() + assert basedir.name == "testarchive" + + def filename(tarinfo): + path_root, _ = path.splitext(tarinfo.name) + return path.split(path_root)[-1] + + assert filename(file1) == "testarchive" + assert filename(file2) == "testarchive" + + +def test_fileobj_not_closed(test_sigmffile): + with tempfile.NamedTemporaryFile() as t: + test_sigmffile.archive(fileobj=t) + assert not t.file.closed + + +def test_unwritable_fileobj_throws_fileerror(test_sigmffile): + with tempfile.NamedTemporaryFile(mode="rb") as t: + with pytest.raises(error.SigMFFileError): + test_sigmffile.archive(fileobj=t) + + +def test_unwritable_name_throws_fileerror(test_sigmffile): + unwritable_file = "/root/unwritable.sigmf" # assumes root is unwritable + with pytest.raises(error.SigMFFileError): + test_sigmffile.archive(name=unwritable_file) + + +def test_tarfile_layout(test_sigmffile): + with tempfile.NamedTemporaryFile() as t: + sigmf_tarfile = create_test_archive(test_sigmffile, t) + basedir, file1, file2 = sigmf_tarfile.getmembers() + assert tarfile.TarInfo.isdir(basedir) + assert tarfile.TarInfo.isfile(file1) + assert tarfile.TarInfo.isfile(file2) + + +def test_tarfile_names_and_extensions(test_sigmffile): + with tempfile.NamedTemporaryFile() as t: + sigmf_tarfile = create_test_archive(test_sigmffile, t) + basedir, file1, file2 = sigmf_tarfile.getmembers() + archive_name = basedir.name + assert archive_name == path.split(t.name)[-1] + file_extensions = {SIGMF_DATASET_EXT, SIGMF_METADATA_EXT} + + file1_name, file1_ext = path.splitext(file1.name) + assert file1_ext in file_extensions + assert path.split(file1_name)[-1] == archive_name + + file_extensions.remove(file1_ext) + + file2_name, file2_ext = path.splitext(file2.name) + assert path.split(file2_name)[-1] == archive_name + assert file2_ext in file_extensions + + +def test_tarfile_persmissions(test_sigmffile): + with tempfile.NamedTemporaryFile() as t: + sigmf_tarfile = create_test_archive(test_sigmffile, t) + basedir, file1, file2 = sigmf_tarfile.getmembers() + assert basedir.mode == 0o755 + assert file1.mode == 0o644 + assert file2.mode == 0o644 + + +def test_contents(test_sigmffile): + with tempfile.NamedTemporaryFile() as t: + sigmf_tarfile = create_test_archive(test_sigmffile, t) + basedir, file1, file2 = sigmf_tarfile.getmembers() + if file1.name.endswith(SIGMF_METADATA_EXT): + mdfile = file1 + datfile = file2 + else: + mdfile = file2 + datfile = file1 + + bytestream_reader = codecs.getreader("utf-8") # bytes -> str + mdfile_reader = bytestream_reader(sigmf_tarfile.extractfile(mdfile)) + assert json.load(mdfile_reader) == TEST_METADATA + + datfile_reader = sigmf_tarfile.extractfile(datfile) + # calling `fileno` on `tarfile.ExFileObject` throws error (?), but + # np.fromfile requires it, so we need this extra step + data = np.fromstring(datfile_reader.read(), dtype=np.float32) + + assert np.array_equal(data, TEST_FLOAT32_DATA) diff --git a/tests/test_sigmffile.py b/tests/test_sigmffile.py new file mode 100644 index 0000000..64ad6aa --- /dev/null +++ b/tests/test_sigmffile.py @@ -0,0 +1,89 @@ +# Copyright 2017 GNU Radio Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import shutil +import tempfile + +import numpy as np + +from sigmf import sigmffile, utils +from sigmf.sigmffile import SigMFFile + +from .testdata import TEST_FLOAT32_DATA, TEST_METADATA + + +def simulate_capture(sigmf_md, n, capture_len): + start_index = capture_len * n + + capture_md = {"core:time": utils.get_sigmf_iso8601_datetime_now()} + + sigmf_md.add_capture(start_index=start_index, metadata=capture_md) + + annotation_md = { + "core:latitude": 40.0 + 0.0001 * n, + "core:longitude": -105.0 + 0.0001 * n, + } + + sigmf_md.add_annotation(start_index=start_index, + length=capture_len, + metadata=annotation_md) + + +def test_default_constructor(): + SigMFFile() + + +def test_set_non_required_global_field(): + f = SigMFFile() + f.set_global_field('this_is:not_in_the_schema', None) + + +def test_add_capture(): + f = SigMFFile() + f.add_capture(start_index=0, metadata={}) + + +def test_add_annotation(): + f = SigMFFile() + f.add_capture(start_index=0) + m = {"latitude": 40.0, "longitude": -105.0} + f.add_annotation(start_index=0, length=128, metadata=m) + + +def test_fromarchive(test_sigmffile): + tf = tempfile.mkstemp()[1] + td = tempfile.mkdtemp() + archive_path = test_sigmffile.archive(name=tf) + result = sigmffile.fromarchive(archive_path=archive_path, dir=td) + + assert result._metadata == test_sigmffile._metadata == TEST_METADATA + + data = np.fromfile(result.data_file, dtype=np.float32) + assert np.array_equal(data, TEST_FLOAT32_DATA) + + os.remove(tf) + shutil.rmtree(td) + + +def test_add_multiple_captures_and_annotations(): + f = SigMFFile() + for i in range(3): + simulate_capture(f, i, 1024) diff --git a/tests/test_validation.py b/tests/test_validation.py index 64d8d0a..3138f27 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -17,24 +17,25 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -""" -Tests -""" -import sigmf +import pytest + +from sigmf.error import SigMFValidationError +from sigmf.sigmffile import SigMFFile + MD_VALID = """ { "global": { "core:datatype": "cf32", "core:offset": 0, - "core:version": "1.0.0", + "core:version": "0.0.1", "core:license": "CC0", "core:date": "foo", "core:url": "foo", "core:sha512": "69a014f8855058d25b30b1caf4f9d15bb7b38afa26e28b24a63545734e534a861d658eddae1dbc666b33ca1d18c1ca85722f1f2f010703a7dbbef08189a1d0e5" }, - "capture": [ + "captures": [ { "core:sample_start": 0, "core:sampling_rate": 10000000, @@ -64,7 +65,7 @@ "global": { "core:datatype": "cf32" }, - "capture": [ + "captures": [ { "core:sample_start": 10 }, @@ -86,7 +87,7 @@ "global": { "core:datatype": "cf32" }, - "capture": [ + "captures": [ { "core:sample_start": 0 } @@ -104,23 +105,11 @@ } """ -MD_EMPTY = """ -{} -""" def test_valid_data(): - assert sigmf.SigMFFile(MD_VALID).validate() + assert SigMFFile(MD_VALID).validate() -def test_invalid_capture_seq(): - assert not sigmf.SigMFFile(MD_INVALID_SEQUENCE_CAP).validate() - assert not sigmf.SigMFFile(MD_INVALID_SEQUENCE_ANN).validate() - -def test_assert_empty(): - pass -def test_default_constructor(): - sigmf.SigMFFile() - -def test_set_non_required_global_field(): - f = sigmf.SigMFFile() - f.set_global_field('this_is:not_in_the_schema', None) +def test_invalid_capture_seq(): + assert not SigMFFile(MD_INVALID_SEQUENCE_CAP).validate() + assert not SigMFFile(MD_INVALID_SEQUENCE_ANN).validate() diff --git a/tests/testdata.py b/tests/testdata.py new file mode 100644 index 0000000..35267d2 --- /dev/null +++ b/tests/testdata.py @@ -0,0 +1,37 @@ +# flake8: noqa + +# Copyright 2017 GNU Radio Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import numpy as np + + +TEST_FLOAT32_DATA = np.arange(16, dtype=np.float32) + +TEST_METADATA = { + 'annotations': [{'core:sample_count': 16, 'core:sample_start': 0}], + 'captures': [{'core:sample_start': 0}], + 'global': { + 'core:datatype': 'f32', + 'core:sha512': 'f4984219b318894fa7144519185d1ae81ea721c6113243a52b51e444512a39d74cf41a4cec3c5d000bd7277cc71232c04d7a946717497e18619bdbe94bfeadd6', + 'core:version': '0.0.1' + } +} diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..433b33f --- /dev/null +++ b/tox.ini @@ -0,0 +1,17 @@ +[tox] +skip_missing_interpreters = True +envlist = py27, py34, py35, py36, py37 + +[testenv] +usedevelop = True +deps = + pytest + flake8 +commands = + pytest + - flake8 + +[testenv:coverage] +deps = + pytest-cov +commands = py.test --cov-report term-missing --cov=sigmf tests