From a59792e1f028e556806986d9dc555ede8aede320 Mon Sep 17 00:00:00 2001 From: Jamie Morton Date: Sat, 8 Apr 2017 08:17:53 -0700 Subject: [PATCH 01/15] ENH: Adding in hierarchy type --- q2_types/tree/__init__.py | 5 +++-- q2_types/tree/_type.py | 7 ++++++- q2_types/tree/tests/test_type.py | 10 +++++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/q2_types/tree/__init__.py b/q2_types/tree/__init__.py index 34bdbb68..532a7b51 100644 --- a/q2_types/tree/__init__.py +++ b/q2_types/tree/__init__.py @@ -9,9 +9,10 @@ import importlib from ._format import NewickFormat, NewickDirectoryFormat -from ._type import Phylogeny, Rooted, Unrooted +from ._type import Phylogeny, Rooted, Unrooted, Hierarchy __all__ = [ - 'NewickFormat', 'NewickDirectoryFormat', 'Phylogeny', 'Rooted', 'Unrooted'] + 'NewickFormat', 'NewickDirectoryFormat', 'Phylogeny', + 'Rooted', 'Unrooted', 'Hierarchy'] importlib.import_module('q2_types.tree._transformer') diff --git a/q2_types/tree/_type.py b/q2_types/tree/_type.py index 0230cf65..095069a4 100644 --- a/q2_types/tree/_type.py +++ b/q2_types/tree/_type.py @@ -18,7 +18,12 @@ Unrooted = SemanticType('Unrooted', variant_of=Phylogeny.field['type']) -plugin.register_semantic_types(Phylogeny, Rooted, Unrooted) +Hierarchy = SemanticType('Hierarchy') + +plugin.register_semantic_types(Phylogeny, Rooted, Unrooted, Hierarchy) plugin.register_semantic_type_to_format(Phylogeny[Rooted | Unrooted], artifact_format=NewickDirectoryFormat) + +plugin.register_semantic_type_to_format(Hierarchy, + artifact_format=NewickDirectoryFormat) diff --git a/q2_types/tree/tests/test_type.py b/q2_types/tree/tests/test_type.py index fe7976b7..9c630bda 100644 --- a/q2_types/tree/tests/test_type.py +++ b/q2_types/tree/tests/test_type.py @@ -8,7 +8,8 @@ import unittest -from q2_types.tree import Phylogeny, Rooted, Unrooted, NewickDirectoryFormat +from q2_types.tree import (Phylogeny, Rooted, Unrooted, + Hierarchy, NewickDirectoryFormat) from qiime2.plugin.testing import TestPluginBase @@ -24,10 +25,17 @@ def test_rooted_semantic_type_registration(self): def test_unrooted_semantic_type_registration(self): self.assertRegisteredSemanticType(Unrooted) + def test_phylogeny_semantic_type_registration(self): + self.assertRegisteredSemanticType(Hierarchy) + def test_phylogeny_rooted_unrooted_to_newick_dir_fmt_registration(self): self.assertSemanticTypeRegisteredToFormat( Phylogeny[Rooted | Unrooted], NewickDirectoryFormat) + def test_phylogeny_rooted_unrooted_to_newick_dir_fmt_registration(self): + self.assertSemanticTypeRegisteredToFormat( + Hierarchy, NewickDirectoryFormat) + if __name__ == '__main__': unittest.main() From 493dac4f553d80ed23c6db57bb91e39cf81363c4 Mon Sep 17 00:00:00 2001 From: Jamie Morton Date: Fri, 12 May 2017 11:24:08 -0700 Subject: [PATCH 02/15] ENH: Adding type back in --- q2_types/tree/tests/test_type.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/q2_types/tree/tests/test_type.py b/q2_types/tree/tests/test_type.py index 9c630bda..55a20b9e 100644 --- a/q2_types/tree/tests/test_type.py +++ b/q2_types/tree/tests/test_type.py @@ -25,14 +25,14 @@ def test_rooted_semantic_type_registration(self): def test_unrooted_semantic_type_registration(self): self.assertRegisteredSemanticType(Unrooted) - def test_phylogeny_semantic_type_registration(self): + def test_hierarchy_semantic_type_registration(self): self.assertRegisteredSemanticType(Hierarchy) def test_phylogeny_rooted_unrooted_to_newick_dir_fmt_registration(self): self.assertSemanticTypeRegisteredToFormat( Phylogeny[Rooted | Unrooted], NewickDirectoryFormat) - def test_phylogeny_rooted_unrooted_to_newick_dir_fmt_registration(self): + def test_hierarchy_to_newick_dir_fmt_registration(self): self.assertSemanticTypeRegisteredToFormat( Hierarchy, NewickDirectoryFormat) From b621850173c945edea927ad6347a0429fd5749ea Mon Sep 17 00:00:00 2001 From: Jamie Morton Date: Fri, 5 Apr 2019 15:54:02 -0400 Subject: [PATCH 03/15] ENH: Adding in differential type --- q2_types/feature_data/__init__.py | 10 ++++++---- q2_types/feature_data/_format.py | 11 ++++++++++- q2_types/feature_data/_transformer.py | 17 ++++++++++++++++- q2_types/feature_data/_type.py | 12 ++++++++++-- q2_types/feature_data/tests/test_type.py | 12 ++++++++++-- 5 files changed, 52 insertions(+), 10 deletions(-) diff --git a/q2_types/feature_data/__init__.py b/q2_types/feature_data/__init__.py index 89992af1..c29cf1ba 100644 --- a/q2_types/feature_data/__init__.py +++ b/q2_types/feature_data/__init__.py @@ -13,9 +13,11 @@ HeaderlessTSVTaxonomyDirectoryFormat, TSVTaxonomyFormat, TSVTaxonomyDirectoryFormat, DNAFASTAFormat, DNASequencesDirectoryFormat, PairedDNASequencesDirectoryFormat, AlignedDNAFASTAFormat, - AlignedDNASequencesDirectoryFormat) + AlignedDNASequencesDirectoryFormat, DifferentialFormat, + DifferentialDirectoryFormat) from ._type import ( - FeatureData, Taxonomy, Sequence, PairedEndSequence, AlignedSequence) + FeatureData, Taxonomy, Sequence, PairedEndSequence, AlignedSequence, + Differential) # TODO remove these imports when tests are rewritten. Remove from __all__ too from ._transformer import DNAIterator, PairedDNAIterator, AlignedDNAIterator @@ -23,11 +25,11 @@ __all__ = [ 'TaxonomyFormat', 'TaxonomyDirectoryFormat', 'HeaderlessTSVTaxonomyFormat', 'HeaderlessTSVTaxonomyDirectoryFormat', 'TSVTaxonomyFormat', - 'TSVTaxonomyDirectoryFormat', 'DNAFASTAFormat', + 'TSVTaxonomyDirectoryFormat', 'DNAFASTAFormat', 'DifferentialFormat', 'DNASequencesDirectoryFormat', 'PairedDNASequencesDirectoryFormat', 'AlignedDNAFASTAFormat', 'AlignedDNASequencesDirectoryFormat', 'FeatureData', 'Taxonomy', 'Sequence', 'PairedEndSequence', 'AlignedSequence', 'DNAIterator', 'PairedDNAIterator', - 'AlignedDNAIterator'] + 'AlignedDNAIterator', 'Differential', 'DifferentialDirectoryFormat'] importlib.import_module('q2_types.feature_data._transformer') diff --git a/q2_types/feature_data/_format.py b/q2_types/feature_data/_format.py index 64cc5c62..501f8bff 100644 --- a/q2_types/feature_data/_format.py +++ b/q2_types/feature_data/_format.py @@ -185,10 +185,19 @@ def sniff(self): AlignedDNAFASTAFormat) +class DifferentialFormat(model.TextFileFormat): + def validate(*args): + pass + + +DifferentialDirectoryFormat = model.SingleFileDirectoryFormat( + 'DifferentialDirectoryFormat', 'differentials.tsv', DifferentialFormat) + plugin.register_formats( TSVTaxonomyFormat, TSVTaxonomyDirectoryFormat, HeaderlessTSVTaxonomyFormat, HeaderlessTSVTaxonomyDirectoryFormat, TaxonomyFormat, TaxonomyDirectoryFormat, DNAFASTAFormat, DNASequencesDirectoryFormat, PairedDNASequencesDirectoryFormat, - AlignedDNAFASTAFormat, AlignedDNASequencesDirectoryFormat + AlignedDNAFASTAFormat, AlignedDNASequencesDirectoryFormat, + DifferentialFormat, DifferentialDirectoryFormat ) diff --git a/q2_types/feature_data/_transformer.py b/q2_types/feature_data/_transformer.py index e370894d..d5faf996 100644 --- a/q2_types/feature_data/_transformer.py +++ b/q2_types/feature_data/_transformer.py @@ -18,7 +18,7 @@ from ..feature_table import BIOMV210Format from . import (TaxonomyFormat, HeaderlessTSVTaxonomyFormat, TSVTaxonomyFormat, DNAFASTAFormat, PairedDNASequencesDirectoryFormat, - AlignedDNAFASTAFormat) + AlignedDNAFASTAFormat, DifferentialFormat) # Taxonomy format transformers @@ -359,3 +359,18 @@ def _19(data: AlignedDNAIterator) -> AlignedDNAFASTAFormat: @plugin.register_transformer def _33(ff: AlignedDNAFASTAFormat) -> qiime2.Metadata: return _dnafastaformats_to_metadata(ff) + + +# differential types +@plugin.register_transformer +def _222(ff: DifferentialFormat) -> pd.DataFrame: + df = pd.read_csv(str(ff), sep='\t', comment='#', skip_blank_lines=True, + header=0, dtype=object, index_col=0) + return df + + +@plugin.register_transformer +def _223(df: pd.DataFrame) -> DifferentialFormat: + ff = DifferentialFormat() + df.to_csv(str(ff), sep='\t', header=True, index=True) + return ff diff --git a/q2_types/feature_data/_type.py b/q2_types/feature_data/_type.py index 53845f8a..147136e3 100644 --- a/q2_types/feature_data/_type.py +++ b/q2_types/feature_data/_type.py @@ -11,7 +11,8 @@ from ..plugin_setup import plugin from . import (TSVTaxonomyDirectoryFormat, DNASequencesDirectoryFormat, PairedDNASequencesDirectoryFormat, - AlignedDNASequencesDirectoryFormat) + AlignedDNASequencesDirectoryFormat, + DifferentialDirectoryFormat) FeatureData = SemanticType('FeatureData', field_names='type') @@ -26,8 +27,13 @@ AlignedSequence = SemanticType('AlignedSequence', variant_of=FeatureData.field['type']) +Differential = SemanticType('Differential', + variant_of=FeatureData.field['type']) + plugin.register_semantic_types(FeatureData, Taxonomy, Sequence, - PairedEndSequence, AlignedSequence) + PairedEndSequence, AlignedSequence, + Differential) + plugin.register_semantic_type_to_format( FeatureData[Taxonomy], @@ -41,3 +47,5 @@ plugin.register_semantic_type_to_format( FeatureData[AlignedSequence], artifact_format=AlignedDNASequencesDirectoryFormat) +plugin.register_semantic_type_to_format( + FeatureData[Differential], DifferentialDirectoryFormat) diff --git a/q2_types/feature_data/tests/test_type.py b/q2_types/feature_data/tests/test_type.py index 09fb6744..6712bae0 100644 --- a/q2_types/feature_data/tests/test_type.py +++ b/q2_types/feature_data/tests/test_type.py @@ -10,8 +10,9 @@ from q2_types.feature_data import ( FeatureData, Taxonomy, Sequence, PairedEndSequence, AlignedSequence, - TSVTaxonomyDirectoryFormat, DNASequencesDirectoryFormat, - PairedDNASequencesDirectoryFormat, AlignedDNASequencesDirectoryFormat + Differential, TSVTaxonomyDirectoryFormat, DNASequencesDirectoryFormat, + DifferentialDirectoryFormat, PairedDNASequencesDirectoryFormat, + AlignedDNASequencesDirectoryFormat ) from qiime2.plugin.testing import TestPluginBase @@ -34,6 +35,13 @@ def test_paired_end_sequence_semantic_type_registration(self): def test_aligned_sequence_semantic_type_registration(self): self.assertRegisteredSemanticType(AlignedSequence) + def test_differential_semantic_type_registration(self): + self.assertRegisteredSemanticType(AlignedSequence) + + def test_differential_semantic_type_to_format_registration(self): + self.assertSemanticTypeRegisteredToFormat( + FeatureData[Differential], DifferentialDirectoryFormat) + def test_taxonomy_semantic_type_to_format_registration(self): self.assertSemanticTypeRegisteredToFormat( FeatureData[Taxonomy], TSVTaxonomyDirectoryFormat) From 0a1aa261ecbc2224c2f73fd66258f260f7769061 Mon Sep 17 00:00:00 2001 From: Jamie Morton Date: Mon, 8 Apr 2019 15:30:14 -0400 Subject: [PATCH 04/15] TST: adding in transformer test --- .../feature_data/tests/data/differentials.tsv | 201 ++++++++++++++++++ .../feature_data/tests/test_transformer.py | 27 ++- 2 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 q2_types/feature_data/tests/data/differentials.tsv diff --git a/q2_types/feature_data/tests/data/differentials.tsv b/q2_types/feature_data/tests/data/differentials.tsv new file mode 100644 index 00000000..055e1819 --- /dev/null +++ b/q2_types/feature_data/tests/data/differentials.tsv @@ -0,0 +1,201 @@ + effect +F0 -0.910182258821735 +F1 1.01418002973925 +F2 1.02456128258909 +F3 -0.74363992043225 +F4 1.29823896534823 +F5 -1.12965055281585 +F6 -0.522401797448688 +F7 0.327560711072239 +F8 -1.3738693465664802 +F9 -0.7847891526325621 +F10 -0.280063201878434 +F11 -0.251269847578052 +F12 1.24602780723028 +F13 0.665734866338239 +F14 -0.889042985114811 +F15 -0.811956802515126 +F16 0.11591764582945001 +F17 -0.156195990858492 +F18 -0.965770633683909 +F19 0.8035240337800391 +F20 0.680306950765235 +F21 -0.688388077896823 +F22 0.7949653982850671 +F23 -1.11912925367142 +F24 -1.1059356352774599 +F25 0.678934047810573 +F26 -0.937189288219405 +F27 -0.9997301931164799 +F28 -0.31799033232181 +F29 -0.66141741897569 +F30 0.550511528315366 +F31 -0.9325448302932858 +F32 -0.34253881283785104 +F33 1.52951044529528 +F34 -0.0322228434081009 +F35 -1.3936520437793 +F36 0.8687222580720692 +F37 0.6823592821191031 +F38 1.37480393072457 +F39 0.239227247696375 +F40 -0.9569262517773199 +F41 1.00830368417208 +F42 0.8288761564573699 +F43 -1.3476440812435702 +F44 0.529025296092455 +F45 -0.340145172805159 +F46 0.964756233144604 +F47 -0.7310281367702399 +F48 0.5383478597420129 +F49 0.29953220850051193 +F50 1.00798009601963 +F51 -0.0132115389102393 +F52 -0.27891571297054396 +F53 0.06277454002648 +F54 0.35167464286304495 +F55 0.137951639939762 +F56 1.33461560518154 +F57 0.43628478179779 +F58 1.11453590606852 +F59 1.2986006866580801 +F60 0.0955549779502926 +F61 0.49599891027325305 +F62 -0.7833488766142149 +F63 -0.18096581167667 +F64 1.30235947734143 +F65 0.280013344538779 +F66 1.30046727244606 +F67 -1.06463133806824 +F68 -1.2593810900401499 +F69 0.264024004173156 +F70 -1.2529907420368602 +F71 1.4937353010846999 +F72 0.970455224810024 +F73 -1.23960496737832 +F74 -0.20356523119618397 +F75 -0.450539610910928 +F76 0.411462370457529 +F77 -0.309713881057648 +F78 0.9140794729087809 +F79 1.3144154384998101 +F80 -0.38732439170956395 +F81 -1.2144940378668603 +F82 0.981754295455261 +F83 0.0289330595561297 +F84 1.2431198940841899 +F85 -0.24182271418096699 +F86 0.76870209333065 +F87 0.23851537911168602 +F88 -0.338651978123051 +F89 -0.4482656135303739 +F90 0.5279056466564099 +F91 1.37807956999098 +F92 1.0431085962714801 +F93 -0.117482004574065 +F94 0.6232413973258649 +F95 0.00403130502610116 +F96 -1.34598938813891 +F97 0.45497065813385906 +F98 1.0192013773894302 +F99 -0.291756302796704 +F100 -1.12688813864611 +F101 0.557769384655089 +F102 1.1975908860910902 +F103 1.15355341444875 +F104 0.946531581472272 +F105 1.2260026125988501 +F106 1.0261210015913198 +F107 0.841857769084618 +F108 -0.4019324938447151 +F109 0.19764363305971105 +F110 -0.149480290579409 +F111 0.6408321521409511 +F112 -0.966639287078902 +F113 1.16738708670172 +F114 0.174326207230448 +F115 -0.5695094365752371 +F116 1.20806940422198 +F117 -1.24543884840423 +F118 -1.15412531725971 +F119 -0.8296471214736691 +F120 0.736529671228746 +F121 -0.660047869199522 +F122 0.9046402016334041 +F123 -0.5986532323341109 +F124 0.691079988752513 +F125 -0.562592354003712 +F126 1.01123956644237 +F127 -0.41423096856367103 +F128 0.619238453979786 +F129 -0.620591765205541 +F130 -1.27255930040442 +F131 -0.35123591295784495 +F132 1.00467461529444 +F133 -1.1354110648325901 +F134 -1.26161085240342 +F135 -0.38659519790583097 +F136 0.872749204870088 +F137 -0.810751128748022 +F138 -0.8197848282048691 +F139 0.20047162644758898 +F140 -0.604549610882948 +F141 0.228624680177828 +F142 1.28609633871252 +F143 -0.1679362832488 +F144 -0.257647674436839 +F145 0.74101019223122 +F146 0.13268740966516301 +F147 -1.09538523785539 +F148 -0.45207411435922207 +F149 -1.1796888761929298 +F150 1.07271995122987 +F151 -0.219846098067332 +F152 0.529493500001223 +F153 0.6270798132806121 +F154 -1.01527420105315 +F155 0.746112185723826 +F156 -0.686265054564927 +F157 -1.3326095821448702 +F158 -0.37778796821367205 +F159 -1.13401841717297 +F160 -0.296439244074386 +F161 0.40847742352928296 +F162 1.27982997837018 +F163 -0.7134413622708241 +F164 -0.349087095579928 +F165 0.6163069356830501 +F166 0.217564847263855 +F167 -0.242902200286938 +F168 -1.01430232660254 +F169 -0.9196469128753921 +F170 -0.19937212012607394 +F171 -1.31129333739411 +F172 -0.721872176785536 +F173 -1.12678232313294 +F174 1.0904935742820798 +F175 -1.17644925652159 +F176 1.25195089868697 +F177 -0.5994841508457149 +F178 -1.1264277507429599 +F179 -0.9273610707857479 +F180 0.852558007271541 +F181 0.740604316851619 +F182 -0.504950438684657 +F183 0.358858461465484 +F184 -0.0652057793033983 +F185 -0.210512467576898 +F186 0.15407114587584303 +F187 0.7501398636830259 +F188 -1.3840507191620501 +F189 -1.3863784044176497 +F190 0.35465073693029203 +F191 -0.0527678764658988 +F192 1.04452771030868 +F193 -1.0105215476872 +F194 -0.6098963926022061 +F195 1.24019945405587 +F196 -1.2486190178628 +F197 0.28873133501175297 +F198 -0.9444328759187891 +F199 -0.308547237330663 diff --git a/q2_types/feature_data/tests/test_transformer.py b/q2_types/feature_data/tests/test_transformer.py index ba0dc2a3..a07e1bf2 100644 --- a/q2_types/feature_data/tests/test_transformer.py +++ b/q2_types/feature_data/tests/test_transformer.py @@ -21,7 +21,7 @@ TaxonomyFormat, HeaderlessTSVTaxonomyFormat, TSVTaxonomyFormat, DNAFASTAFormat, DNAIterator, PairedDNAIterator, PairedDNASequencesDirectoryFormat, AlignedDNAFASTAFormat, - AlignedDNAIterator + DifferentialFormat, AlignedDNAIterator ) from q2_types.feature_data._transformer import ( _taxonomy_formats_to_dataframe, _dataframe_to_tsv_taxonomy_format) @@ -656,5 +656,30 @@ def test_aligned_dnafasta_format_to_metadata(self): self.assertEqual(exp, obs) +class TestDifferentialTransformer(TestPluginBase): + package = 'q2_types.feature_data.tests' + + def test_differential_to_df(self): + + _, obs = self.transform_format(DifferentialFormat, pd.DataFrame, + filename='differentials.tsv') + + # sniff to see if the first 4 feature ids are the same + exp = ['F0', 'F1', 'F2', 'F3'] + obs = list(obs.index[:4]) + self.assertListEqual(exp, obs) + + def test_df_to_differential(self): + transformer = self.get_transformer(pd.DataFrame, DifferentialFormat) + + index = pd.Index(['SEQUENCE1', 'SEQUENCE2', 'SEQUENCE3']) + input = pd.Series([-1.3, 0.1, 1.2], index=index, dtype=float) + + obs = transformer(input) + + self.assertIsInstance(obs, DifferentialFormat) + + + if __name__ == '__main__': unittest.main() From 9948f1b3ead0543059feef89eee45090a6ea6d72 Mon Sep 17 00:00:00 2001 From: Jamie Morton Date: Mon, 8 Apr 2019 15:32:46 -0400 Subject: [PATCH 05/15] STY: flake8 thanks --- q2_types/feature_data/tests/test_format.py | 22 ++++++++++++++++++- .../feature_data/tests/test_transformer.py | 1 - 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/q2_types/feature_data/tests/test_format.py b/q2_types/feature_data/tests/test_format.py index a18a29f7..e72b81fe 100644 --- a/q2_types/feature_data/tests/test_format.py +++ b/q2_types/feature_data/tests/test_format.py @@ -16,7 +16,7 @@ HeaderlessTSVTaxonomyDirectoryFormat, TSVTaxonomyFormat, TSVTaxonomyDirectoryFormat, DNAFASTAFormat, DNASequencesDirectoryFormat, PairedDNASequencesDirectoryFormat, AlignedDNAFASTAFormat, - AlignedDNASequencesDirectoryFormat + AlignedDNASequencesDirectoryFormat, DifferentialDirectoryFormat ) from qiime2.plugin.testing import TestPluginBase from qiime2.plugin import ValidationError @@ -132,6 +132,13 @@ def test_tsv_taxonomy_directory_format(self): format = TSVTaxonomyDirectoryFormat(self.temp_dir.name, mode='r') + filepath = self.get_data_path('aligned-dna-sequences.fasta') + temp_dir = self.temp_dir.name + shutil.copy(filepath, + os.path.join(temp_dir, 'aligned-dna-sequences.fasta')) + format = AlignedDNASequencesDirectoryFormat(temp_dir, mode='r') + + format.validate() format.validate() @@ -203,5 +210,18 @@ def test_aligned_dna_sequences_directory_format(self): format.validate() +class TestDifferentialFormat(TestPluginBase): + package = 'q2_types.feature_data.tests' + + def test_differential_format(self): + filepath = self.get_data_path('differentials.tsv') + temp_dir = self.temp_dir.name + shutil.copy(filepath, + os.path.join(temp_dir, 'differentials.tsv')) + format = DifferentialDirectoryFormat(temp_dir, mode='r') + + format.validate() + + if __name__ == '__main__': unittest.main() diff --git a/q2_types/feature_data/tests/test_transformer.py b/q2_types/feature_data/tests/test_transformer.py index a07e1bf2..b65a149b 100644 --- a/q2_types/feature_data/tests/test_transformer.py +++ b/q2_types/feature_data/tests/test_transformer.py @@ -680,6 +680,5 @@ def test_df_to_differential(self): self.assertIsInstance(obs, DifferentialFormat) - if __name__ == '__main__': unittest.main() From 56e1161cf88572373dd83b61316e40240e1ccfeb Mon Sep 17 00:00:00 2001 From: Jamie Morton Date: Tue, 9 Apr 2019 13:57:38 -0400 Subject: [PATCH 06/15] REV: revert copy pasta --- q2_types/feature_data/tests/test_format.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/q2_types/feature_data/tests/test_format.py b/q2_types/feature_data/tests/test_format.py index e72b81fe..548af8f0 100644 --- a/q2_types/feature_data/tests/test_format.py +++ b/q2_types/feature_data/tests/test_format.py @@ -131,14 +131,6 @@ def test_tsv_taxonomy_directory_format(self): os.path.join(self.temp_dir.name, 'taxonomy.tsv')) format = TSVTaxonomyDirectoryFormat(self.temp_dir.name, mode='r') - - filepath = self.get_data_path('aligned-dna-sequences.fasta') - temp_dir = self.temp_dir.name - shutil.copy(filepath, - os.path.join(temp_dir, 'aligned-dna-sequences.fasta')) - format = AlignedDNASequencesDirectoryFormat(temp_dir, mode='r') - - format.validate() format.validate() From a15a4f08f576caaf387598af60e4adae7b50271e Mon Sep 17 00:00:00 2001 From: Jamie Morton Date: Thu, 23 May 2019 16:47:27 -0400 Subject: [PATCH 07/15] ENH: adding new files / checks / transformers. Tests to come --- q2_types/feature_data/_format.py | 25 ++- q2_types/feature_data/_transformer.py | 11 +- .../tests/data/bad_differential.tsv | 201 ++++++++++++++++++ .../feature_data/tests/data/differentials.tsv | 2 +- .../tests/data/empty_differential.tsv | 201 ++++++++++++++++++ .../tests/data/inf_differential.tsv | 201 ++++++++++++++++++ 6 files changed, 634 insertions(+), 7 deletions(-) create mode 100644 q2_types/feature_data/tests/data/bad_differential.tsv create mode 100644 q2_types/feature_data/tests/data/empty_differential.tsv create mode 100644 q2_types/feature_data/tests/data/inf_differential.tsv diff --git a/q2_types/feature_data/_format.py b/q2_types/feature_data/_format.py index 501f8bff..615dbbc6 100644 --- a/q2_types/feature_data/_format.py +++ b/q2_types/feature_data/_format.py @@ -187,7 +187,30 @@ def sniff(self): class DifferentialFormat(model.TextFileFormat): def validate(*args): - pass + + try: + md = qiime2.Metadata.load(str(self)) + except qiime2.metadata.MetadataFileError as md_exc: + raise ValidationError(md_exc) from md_exc + + md = md.to_dataframe() + if len(md.columns) == 0: + raise ValidationError( + ValueError( + ('Differential format must contain' + 'at least 1 column') + ) + ) + + types = md.dtypes + for t in types: + if t != np.float64 or t != np.float32: + raise ValidationError( + ValueError( + ('Differential types must only contain ' + 'continuously valued quantities') + ) + ) DifferentialDirectoryFormat = model.SingleFileDirectoryFormat( diff --git a/q2_types/feature_data/_transformer.py b/q2_types/feature_data/_transformer.py index d5faf996..0e01d445 100644 --- a/q2_types/feature_data/_transformer.py +++ b/q2_types/feature_data/_transformer.py @@ -364,13 +364,14 @@ def _33(ff: AlignedDNAFASTAFormat) -> qiime2.Metadata: # differential types @plugin.register_transformer def _222(ff: DifferentialFormat) -> pd.DataFrame: - df = pd.read_csv(str(ff), sep='\t', comment='#', skip_blank_lines=True, - header=0, dtype=object, index_col=0) - return df + return Metadata.load(str(ff)).to_dataframe() +@plugin.register_transformer +def _223(ff: DifferentialFormat) -> Metadata: + return Metadata.load(str(ff)) @plugin.register_transformer -def _223(df: pd.DataFrame) -> DifferentialFormat: +def _224(df: pd.DataFrame) -> DifferentialFormat: ff = DifferentialFormat() - df.to_csv(str(ff), sep='\t', header=True, index=True) + Metadata(data).save(str(ff)) return ff diff --git a/q2_types/feature_data/tests/data/bad_differential.tsv b/q2_types/feature_data/tests/data/bad_differential.tsv new file mode 100644 index 00000000..c06291cc --- /dev/null +++ b/q2_types/feature_data/tests/data/bad_differential.tsv @@ -0,0 +1,201 @@ +featureid effect bad_effect +F0 -0.910182258821735 0 +F1 1.01418002973925 0 +F2 1.02456128258909 0 +F3 -0.74363992043225 0 +F4 1.29823896534823 0 +F5 -1.12965055281585 0 +F6 -0.522401797448688 0 +F7 0.327560711072239 0 +F8 -1.3738693465664802 0 +F9 -0.7847891526325621 0 +F10 -0.280063201878434 0 +F11 -0.251269847578052 0 +F12 1.24602780723028 0 +F13 0.665734866338239 0 +F14 -0.889042985114811 0 +F15 -0.811956802515126 0 +F16 0.11591764582945001 0 +F17 -0.156195990858492 0 +F18 -0.965770633683909 0 +F19 0.8035240337800391 0 +F20 0.680306950765235 0 +F21 -0.688388077896823 0 +F22 0.7949653982850671 0 +F23 -1.11912925367142 0 +F24 -1.1059356352774599 0 +F25 0.678934047810573 0 +F26 -0.937189288219405 0 +F27 -0.9997301931164799 0 +F28 -0.31799033232181 0 +F29 -0.66141741897569 0 +F30 0.550511528315366 0 +F31 -0.9325448302932858 0 +F32 -0.34253881283785104 0 +F33 1.52951044529528 0 +F34 -0.0322228434081009 0 +F35 -1.3936520437793 0 +F36 0.8687222580720692 0 +F37 0.6823592821191031 0 +F38 1.37480393072457 0 +F39 0.239227247696375 0 +F40 -0.9569262517773199 0 +F41 1.00830368417208 0 +F42 0.8288761564573699 0 +F43 -1.3476440812435702 0 +F44 0.529025296092455 0 +F45 -0.340145172805159 0 +F46 0.964756233144604 0 +F47 -0.7310281367702399 0 +F48 0.5383478597420129 0 +F49 0.29953220850051193 0 +F50 1.00798009601963 0 +F51 -0.0132115389102393 0 +F52 -0.27891571297054396 0 +F53 0.06277454002648 0 +F54 0.3516746428630449 0 +F55 0.137951639939762 0 +F56 1.33461560518154 0 +F57 0.43628478179779 0 +F58 1.11453590606852 0 +F59 1.2986006866580801 0 +F60 0.0955549779502926 0 +F61 0.49599891027325305 0 +F62 -0.7833488766142149 0 +F63 -0.18096581167667 0 +F64 1.30235947734143 0 +F65 0.280013344538779 0 +F66 1.30046727244606 0 +F67 -1.06463133806824 0 +F68 -1.2593810900401499 0 +F69 0.264024004173156 0 +F70 -1.2529907420368602 0 +F71 1.4937353010846999 0 +F72 0.970455224810024 0 +F73 -1.23960496737832 0 +F74 -0.20356523119618397 0 +F75 -0.450539610910928 0 +F76 0.411462370457529 0 +F77 -0.309713881057648 0 +F78 0.9140794729087808 0 +F79 1.3144154384998101 0 +F80 -0.38732439170956395 0 +F81 -1.2144940378668605 0 +F82 0.981754295455261 0 +F83 0.0289330595561297 0 +F84 1.2431198940841899 0 +F85 -0.24182271418096696 0 +F86 0.76870209333065 0 +F87 0.23851537911168602 0 +F88 -0.338651978123051 0 +F89 -0.4482656135303739 0 +F90 0.5279056466564099 0 +F91 1.37807956999098 0 +F92 1.0431085962714801 0 +F93 -0.117482004574065 0 +F94 0.6232413973258649 0 +F95 0.00403130502610116 0 +F96 -1.34598938813891 0 +F97 0.4549706581338591 0 +F98 1.0192013773894302 0 +F99 -0.291756302796704 0 +F100 -1.12688813864611 0 +F101 0.557769384655089 0 +F102 1.1975908860910902 0 +F103 1.15355341444875 0 +F104 0.946531581472272 0 +F105 1.2260026125988501 0 +F106 1.0261210015913198 0 +F107 0.841857769084618 0 +F108 -0.4019324938447151 0 +F109 0.19764363305971105 0 +F110 -0.149480290579409 0 +F111 0.6408321521409511 0 +F112 -0.966639287078902 0 +F113 1.16738708670172 0 +F114 0.174326207230448 0 +F115 -0.5695094365752371 0 +F116 1.20806940422198 0 +F117 -1.24543884840423 0 +F118 -1.15412531725971 0 +F119 -0.8296471214736691 0 +F120 0.736529671228746 0 +F121 -0.660047869199522 0 +F122 0.9046402016334041 0 +F123 -0.5986532323341109 0 +F124 0.691079988752513 0 +F125 -0.562592354003712 0 +F126 1.01123956644237 0 +F127 -0.41423096856367103 0 +F128 0.619238453979786 0 +F129 -0.620591765205541 0 +F130 -1.27255930040442 0 +F131 -0.3512359129578449 0 +F132 1.00467461529444 0 +F133 -1.1354110648325901 0 +F134 -1.26161085240342 0 +F135 -0.38659519790583097 0 +F136 0.872749204870088 0 +F137 -0.810751128748022 0 +F138 -0.8197848282048691 0 +F139 0.20047162644758892 0 +F140 -0.604549610882948 0 +F141 0.228624680177828 0 +F142 1.28609633871252 0 +F143 -0.1679362832488 0 +F144 -0.257647674436839 0 +F145 0.74101019223122 0 +F146 0.13268740966516301 0 +F147 -1.09538523785539 0 +F148 -0.4520741143592221 0 +F149 -1.1796888761929298 0 +F150 1.07271995122987 0 +F151 -0.219846098067332 0 +F152 0.529493500001223 0 +F153 0.6270798132806121 0 +F154 -1.01527420105315 0 +F155 0.746112185723826 0 +F156 -0.686265054564927 0 +F157 -1.3326095821448702 0 +F158 -0.3777879682136721 0 +F159 -1.13401841717297 0 +F160 -0.296439244074386 0 +F161 0.40847742352928296 0 +F162 1.27982997837018 0 +F163 -0.7134413622708241 0 +F164 -0.349087095579928 0 +F165 0.6163069356830501 0 +F166 0.217564847263855 0 +F167 -0.242902200286938 0 +F168 -1.01430232660254 0 +F169 -0.9196469128753921 0 +F170 -0.19937212012607394 0 +F171 -1.31129333739411 0 +F172 -0.721872176785536 0 +F173 -1.12678232313294 0 +F174 1.0904935742820798 0 +F175 -1.17644925652159 0 +F176 1.25195089868697 0 +F177 -0.5994841508457149 0 +F178 -1.1264277507429599 0 +F179 -0.9273610707857479 0 +F180 0.852558007271541 0 +F181 0.740604316851619 0 +F182 -0.504950438684657 0 +F183 0.358858461465484 0 +F184 -0.0652057793033983 0 +F185 -0.210512467576898 0 +F186 0.15407114587584306 0 +F187 0.7501398636830259 0 +F188 -1.3840507191620501 0 +F189 -1.3863784044176497 0 +F190 0.35465073693029203 0 +F191 -0.0527678764658988 0 +F192 1.04452771030868 0 +F193 -1.0105215476872 0 +F194 -0.6098963926022061 0 +F195 1.24019945405587 0 +F196 -1.2486190178628 0 +F197 0.28873133501175297 0 +F198 -0.9444328759187892 0 +F199 -0.308547237330663 0 diff --git a/q2_types/feature_data/tests/data/differentials.tsv b/q2_types/feature_data/tests/data/differentials.tsv index 055e1819..2c637a4f 100644 --- a/q2_types/feature_data/tests/data/differentials.tsv +++ b/q2_types/feature_data/tests/data/differentials.tsv @@ -1,4 +1,4 @@ - effect +featureid effect F0 -0.910182258821735 F1 1.01418002973925 F2 1.02456128258909 diff --git a/q2_types/feature_data/tests/data/empty_differential.tsv b/q2_types/feature_data/tests/data/empty_differential.tsv new file mode 100644 index 00000000..f4895cd6 --- /dev/null +++ b/q2_types/feature_data/tests/data/empty_differential.tsv @@ -0,0 +1,201 @@ +featureid +F0 +F1 +F2 +F3 +F4 +F5 +F6 +F7 +F8 +F9 +F10 +F11 +F12 +F13 +F14 +F15 +F16 +F17 +F18 +F19 +F20 +F21 +F22 +F23 +F24 +F25 +F26 +F27 +F28 +F29 +F30 +F31 +F32 +F33 +F34 +F35 +F36 +F37 +F38 +F39 +F40 +F41 +F42 +F43 +F44 +F45 +F46 +F47 +F48 +F49 +F50 +F51 +F52 +F53 +F54 +F55 +F56 +F57 +F58 +F59 +F60 +F61 +F62 +F63 +F64 +F65 +F66 +F67 +F68 +F69 +F70 +F71 +F72 +F73 +F74 +F75 +F76 +F77 +F78 +F79 +F80 +F81 +F82 +F83 +F84 +F85 +F86 +F87 +F88 +F89 +F90 +F91 +F92 +F93 +F94 +F95 +F96 +F97 +F98 +F99 +F100 +F101 +F102 +F103 +F104 +F105 +F106 +F107 +F108 +F109 +F110 +F111 +F112 +F113 +F114 +F115 +F116 +F117 +F118 +F119 +F120 +F121 +F122 +F123 +F124 +F125 +F126 +F127 +F128 +F129 +F130 +F131 +F132 +F133 +F134 +F135 +F136 +F137 +F138 +F139 +F140 +F141 +F142 +F143 +F144 +F145 +F146 +F147 +F148 +F149 +F150 +F151 +F152 +F153 +F154 +F155 +F156 +F157 +F158 +F159 +F160 +F161 +F162 +F163 +F164 +F165 +F166 +F167 +F168 +F169 +F170 +F171 +F172 +F173 +F174 +F175 +F176 +F177 +F178 +F179 +F180 +F181 +F182 +F183 +F184 +F185 +F186 +F187 +F188 +F189 +F190 +F191 +F192 +F193 +F194 +F195 +F196 +F197 +F198 +F199 diff --git a/q2_types/feature_data/tests/data/inf_differential.tsv b/q2_types/feature_data/tests/data/inf_differential.tsv new file mode 100644 index 00000000..db5db536 --- /dev/null +++ b/q2_types/feature_data/tests/data/inf_differential.tsv @@ -0,0 +1,201 @@ +featureid effect +F0 inf +F1 1.01418002973925 +F2 1.02456128258909 +F3 -0.74363992043225 +F4 1.29823896534823 +F5 -1.12965055281585 +F6 -0.522401797448688 +F7 0.327560711072239 +F8 -1.3738693465664802 +F9 -0.7847891526325621 +F10 -0.280063201878434 +F11 -0.251269847578052 +F12 1.24602780723028 +F13 0.665734866338239 +F14 -0.889042985114811 +F15 -0.811956802515126 +F16 0.11591764582945 +F17 -0.156195990858492 +F18 -0.965770633683909 +F19 0.8035240337800391 +F20 0.680306950765235 +F21 -0.688388077896823 +F22 0.7949653982850671 +F23 -1.11912925367142 +F24 -1.10593563527746 +F25 0.678934047810573 +F26 -0.937189288219405 +F27 -0.99973019311648 +F28 -0.31799033232181 +F29 -0.66141741897569 +F30 0.550511528315366 +F31 -0.9325448302932858 +F32 -0.34253881283785104 +F33 1.52951044529528 +F34 -0.0322228434081009 +F35 -1.3936520437793 +F36 0.8687222580720692 +F37 0.6823592821191031 +F38 1.37480393072457 +F39 0.239227247696375 +F40 -0.95692625177732 +F41 1.00830368417208 +F42 0.8288761564573699 +F43 -1.3476440812435702 +F44 0.529025296092455 +F45 -0.340145172805159 +F46 0.964756233144604 +F47 -0.7310281367702399 +F48 0.5383478597420129 +F49 0.29953220850051193 +F50 1.00798009601963 +F51 -0.0132115389102393 +F52 -0.278915712970544 +F53 0.06277454002648 +F54 0.3516746428630449 +F55 0.137951639939762 +F56 1.33461560518154 +F57 0.43628478179779 +F58 1.11453590606852 +F59 1.29860068665808 +F60 0.0955549779502926 +F61 0.495998910273253 +F62 -0.7833488766142149 +F63 -0.18096581167667 +F64 1.30235947734143 +F65 0.280013344538779 +F66 1.30046727244606 +F67 -1.06463133806824 +F68 -1.25938109004015 +F69 0.264024004173156 +F70 -1.2529907420368602 +F71 1.4937353010847 +F72 0.970455224810024 +F73 -1.23960496737832 +F74 -0.203565231196184 +F75 -0.450539610910928 +F76 0.411462370457529 +F77 -0.309713881057648 +F78 0.9140794729087808 +F79 1.31441543849981 +F80 -0.387324391709564 +F81 -1.2144940378668605 +F82 0.981754295455261 +F83 0.0289330595561297 +F84 1.24311989408419 +F85 -0.24182271418096696 +F86 0.76870209333065 +F87 0.238515379111686 +F88 -0.338651978123051 +F89 -0.4482656135303739 +F90 0.5279056466564099 +F91 1.37807956999098 +F92 1.04310859627148 +F93 -0.117482004574065 +F94 0.6232413973258649 +F95 0.00403130502610116 +F96 -1.34598938813891 +F97 0.4549706581338591 +F98 1.0192013773894302 +F99 -0.291756302796704 +F100 -1.12688813864611 +F101 0.557769384655089 +F102 1.1975908860910902 +F103 1.15355341444875 +F104 0.946531581472272 +F105 1.22600261259885 +F106 1.0261210015913198 +F107 0.841857769084618 +F108 -0.4019324938447151 +F109 0.19764363305971105 +F110 -0.149480290579409 +F111 0.6408321521409511 +F112 -0.966639287078902 +F113 1.16738708670172 +F114 0.174326207230448 +F115 -0.5695094365752371 +F116 1.20806940422198 +F117 -1.24543884840423 +F118 -1.15412531725971 +F119 -0.8296471214736691 +F120 0.736529671228746 +F121 -0.660047869199522 +F122 0.904640201633404 +F123 -0.5986532323341109 +F124 0.691079988752513 +F125 -0.562592354003712 +F126 1.01123956644237 +F127 -0.41423096856367103 +F128 0.619238453979786 +F129 -0.620591765205541 +F130 -1.27255930040442 +F131 -0.3512359129578449 +F132 1.00467461529444 +F133 -1.13541106483259 +F134 -1.26161085240342 +F135 -0.38659519790583097 +F136 0.872749204870088 +F137 -0.810751128748022 +F138 -0.8197848282048691 +F139 0.20047162644758895 +F140 -0.604549610882948 +F141 0.228624680177828 +F142 1.28609633871252 +F143 -0.1679362832488 +F144 -0.257647674436839 +F145 0.74101019223122 +F146 0.132687409665163 +F147 -1.09538523785539 +F148 -0.4520741143592221 +F149 -1.1796888761929298 +F150 1.07271995122987 +F151 -0.219846098067332 +F152 0.529493500001223 +F153 0.6270798132806121 +F154 -1.01527420105315 +F155 0.746112185723826 +F156 -0.686265054564927 +F157 -1.3326095821448702 +F158 -0.3777879682136721 +F159 -1.13401841717297 +F160 -0.296439244074386 +F161 0.40847742352928296 +F162 1.27982997837018 +F163 -0.7134413622708241 +F164 -0.349087095579928 +F165 0.6163069356830501 +F166 0.217564847263855 +F167 -0.242902200286938 +F168 -1.01430232660254 +F169 -0.919646912875392 +F170 -0.19937212012607394 +F171 -1.31129333739411 +F172 -0.721872176785536 +F173 -1.12678232313294 +F174 1.0904935742820798 +F175 -1.17644925652159 +F176 1.25195089868697 +F177 -0.5994841508457149 +F178 -1.12642775074296 +F179 -0.927361070785748 +F180 0.852558007271541 +F181 0.740604316851619 +F182 -0.504950438684657 +F183 0.358858461465484 +F184 -0.0652057793033983 +F185 -0.210512467576898 +F186 0.15407114587584306 +F187 0.7501398636830259 +F188 -1.38405071916205 +F189 -1.3863784044176497 +F190 0.35465073693029203 +F191 -0.0527678764658988 +F192 1.04452771030868 +F193 -1.0105215476872 +F194 -0.6098963926022061 +F195 1.24019945405587 +F196 -1.2486190178628 +F197 0.28873133501175297 +F198 -0.9444328759187892 +F199 -0.308547237330663 From f08c61383319904a6e9d2131df4f805bd768931f Mon Sep 17 00:00:00 2001 From: mortonjt Date: Fri, 24 May 2019 12:34:10 -0400 Subject: [PATCH 08/15] TST: tests pass --- q2_types/feature_data/_format.py | 52 ++- q2_types/feature_data/_transformer.py | 8 +- .../feature_data/tests/data/differentials.tsv | 402 +++++++++--------- q2_types/feature_data/tests/test_format.py | 40 ++ 4 files changed, 275 insertions(+), 227 deletions(-) diff --git a/q2_types/feature_data/_format.py b/q2_types/feature_data/_format.py index 615dbbc6..d14bc94b 100644 --- a/q2_types/feature_data/_format.py +++ b/q2_types/feature_data/_format.py @@ -5,9 +5,12 @@ # # The full license is in the file LICENSE, distributed with this software. # ---------------------------------------------------------------------------- - +import numpy as np +import pandas as pd import skbio.io +import qiime2 import qiime2.plugin.model as model +from qiime2.plugin import ValidationError from ..plugin_setup import plugin @@ -186,33 +189,38 @@ def sniff(self): class DifferentialFormat(model.TextFileFormat): - def validate(*args): + def validate(self, *args): try: md = qiime2.Metadata.load(str(self)) except qiime2.metadata.MetadataFileError as md_exc: raise ValidationError(md_exc) from md_exc - md = md.to_dataframe() - if len(md.columns) == 0: - raise ValidationError( - ValueError( - ('Differential format must contain' - 'at least 1 column') - ) - ) - - types = md.dtypes - for t in types: - if t != np.float64 or t != np.float32: - raise ValidationError( - ValueError( - ('Differential types must only contain ' - 'continuously valued quantities') - ) - ) - - + md = md.to_dataframe() + if len(md.columns) == 0: + raise ValueError( + ('Differential format must contain' + 'at least 1 column') + ) + + + types = md.dtypes + for t in types: + if not np.issubdtype(t, np.number): + raise ValueError( + ('Differential types must only contain ' + 'continuously valued quantities') + ) + + + for c in md.columns: + if (np.isinf(md[c].values).sum() > 0 or + pd.isnull(md[c]).sum() > 0): + raise ValueError( + ('Differential contains infs or nans.') + ) + + DifferentialDirectoryFormat = model.SingleFileDirectoryFormat( 'DifferentialDirectoryFormat', 'differentials.tsv', DifferentialFormat) diff --git a/q2_types/feature_data/_transformer.py b/q2_types/feature_data/_transformer.py index 0e01d445..b1066153 100644 --- a/q2_types/feature_data/_transformer.py +++ b/q2_types/feature_data/_transformer.py @@ -364,14 +364,14 @@ def _33(ff: AlignedDNAFASTAFormat) -> qiime2.Metadata: # differential types @plugin.register_transformer def _222(ff: DifferentialFormat) -> pd.DataFrame: - return Metadata.load(str(ff)).to_dataframe() + return qiime2.Metadata.load(str(ff)).to_dataframe() @plugin.register_transformer -def _223(ff: DifferentialFormat) -> Metadata: - return Metadata.load(str(ff)) +def _223(ff: DifferentialFormat) -> qiime2.Metadata: + return qiime2.Metadata.load(str(ff)) @plugin.register_transformer def _224(df: pd.DataFrame) -> DifferentialFormat: ff = DifferentialFormat() - Metadata(data).save(str(ff)) + qiime2.Metadata(data).save(str(ff)) return ff diff --git a/q2_types/feature_data/tests/data/differentials.tsv b/q2_types/feature_data/tests/data/differentials.tsv index 2c637a4f..116da945 100644 --- a/q2_types/feature_data/tests/data/differentials.tsv +++ b/q2_types/feature_data/tests/data/differentials.tsv @@ -1,201 +1,201 @@ -featureid effect -F0 -0.910182258821735 -F1 1.01418002973925 -F2 1.02456128258909 -F3 -0.74363992043225 -F4 1.29823896534823 -F5 -1.12965055281585 -F6 -0.522401797448688 -F7 0.327560711072239 -F8 -1.3738693465664802 -F9 -0.7847891526325621 -F10 -0.280063201878434 -F11 -0.251269847578052 -F12 1.24602780723028 -F13 0.665734866338239 -F14 -0.889042985114811 -F15 -0.811956802515126 -F16 0.11591764582945001 -F17 -0.156195990858492 -F18 -0.965770633683909 -F19 0.8035240337800391 -F20 0.680306950765235 -F21 -0.688388077896823 -F22 0.7949653982850671 -F23 -1.11912925367142 -F24 -1.1059356352774599 -F25 0.678934047810573 -F26 -0.937189288219405 -F27 -0.9997301931164799 -F28 -0.31799033232181 -F29 -0.66141741897569 -F30 0.550511528315366 -F31 -0.9325448302932858 -F32 -0.34253881283785104 -F33 1.52951044529528 -F34 -0.0322228434081009 -F35 -1.3936520437793 -F36 0.8687222580720692 -F37 0.6823592821191031 -F38 1.37480393072457 -F39 0.239227247696375 -F40 -0.9569262517773199 -F41 1.00830368417208 -F42 0.8288761564573699 -F43 -1.3476440812435702 -F44 0.529025296092455 -F45 -0.340145172805159 -F46 0.964756233144604 -F47 -0.7310281367702399 -F48 0.5383478597420129 -F49 0.29953220850051193 -F50 1.00798009601963 -F51 -0.0132115389102393 -F52 -0.27891571297054396 -F53 0.06277454002648 -F54 0.35167464286304495 -F55 0.137951639939762 -F56 1.33461560518154 -F57 0.43628478179779 -F58 1.11453590606852 -F59 1.2986006866580801 -F60 0.0955549779502926 -F61 0.49599891027325305 -F62 -0.7833488766142149 -F63 -0.18096581167667 -F64 1.30235947734143 -F65 0.280013344538779 -F66 1.30046727244606 -F67 -1.06463133806824 -F68 -1.2593810900401499 -F69 0.264024004173156 -F70 -1.2529907420368602 -F71 1.4937353010846999 -F72 0.970455224810024 -F73 -1.23960496737832 -F74 -0.20356523119618397 -F75 -0.450539610910928 -F76 0.411462370457529 -F77 -0.309713881057648 -F78 0.9140794729087809 -F79 1.3144154384998101 -F80 -0.38732439170956395 -F81 -1.2144940378668603 -F82 0.981754295455261 -F83 0.0289330595561297 -F84 1.2431198940841899 -F85 -0.24182271418096699 -F86 0.76870209333065 -F87 0.23851537911168602 -F88 -0.338651978123051 -F89 -0.4482656135303739 -F90 0.5279056466564099 -F91 1.37807956999098 -F92 1.0431085962714801 -F93 -0.117482004574065 -F94 0.6232413973258649 -F95 0.00403130502610116 -F96 -1.34598938813891 -F97 0.45497065813385906 -F98 1.0192013773894302 -F99 -0.291756302796704 -F100 -1.12688813864611 -F101 0.557769384655089 -F102 1.1975908860910902 -F103 1.15355341444875 -F104 0.946531581472272 -F105 1.2260026125988501 -F106 1.0261210015913198 -F107 0.841857769084618 -F108 -0.4019324938447151 -F109 0.19764363305971105 -F110 -0.149480290579409 -F111 0.6408321521409511 -F112 -0.966639287078902 -F113 1.16738708670172 -F114 0.174326207230448 -F115 -0.5695094365752371 -F116 1.20806940422198 -F117 -1.24543884840423 -F118 -1.15412531725971 -F119 -0.8296471214736691 -F120 0.736529671228746 -F121 -0.660047869199522 -F122 0.9046402016334041 -F123 -0.5986532323341109 -F124 0.691079988752513 -F125 -0.562592354003712 -F126 1.01123956644237 -F127 -0.41423096856367103 -F128 0.619238453979786 -F129 -0.620591765205541 -F130 -1.27255930040442 -F131 -0.35123591295784495 -F132 1.00467461529444 -F133 -1.1354110648325901 -F134 -1.26161085240342 -F135 -0.38659519790583097 -F136 0.872749204870088 -F137 -0.810751128748022 -F138 -0.8197848282048691 -F139 0.20047162644758898 -F140 -0.604549610882948 -F141 0.228624680177828 -F142 1.28609633871252 -F143 -0.1679362832488 -F144 -0.257647674436839 -F145 0.74101019223122 -F146 0.13268740966516301 -F147 -1.09538523785539 -F148 -0.45207411435922207 -F149 -1.1796888761929298 -F150 1.07271995122987 -F151 -0.219846098067332 -F152 0.529493500001223 -F153 0.6270798132806121 -F154 -1.01527420105315 -F155 0.746112185723826 -F156 -0.686265054564927 -F157 -1.3326095821448702 -F158 -0.37778796821367205 -F159 -1.13401841717297 -F160 -0.296439244074386 -F161 0.40847742352928296 -F162 1.27982997837018 -F163 -0.7134413622708241 -F164 -0.349087095579928 -F165 0.6163069356830501 -F166 0.217564847263855 -F167 -0.242902200286938 -F168 -1.01430232660254 -F169 -0.9196469128753921 -F170 -0.19937212012607394 -F171 -1.31129333739411 -F172 -0.721872176785536 -F173 -1.12678232313294 -F174 1.0904935742820798 -F175 -1.17644925652159 -F176 1.25195089868697 -F177 -0.5994841508457149 -F178 -1.1264277507429599 -F179 -0.9273610707857479 -F180 0.852558007271541 -F181 0.740604316851619 -F182 -0.504950438684657 -F183 0.358858461465484 -F184 -0.0652057793033983 -F185 -0.210512467576898 -F186 0.15407114587584303 -F187 0.7501398636830259 -F188 -1.3840507191620501 -F189 -1.3863784044176497 -F190 0.35465073693029203 -F191 -0.0527678764658988 -F192 1.04452771030868 -F193 -1.0105215476872 -F194 -0.6098963926022061 -F195 1.24019945405587 -F196 -1.2486190178628 -F197 0.28873133501175297 -F198 -0.9444328759187891 -F199 -0.308547237330663 +featureid effect confounder +F0 -0.910182258821735 -0.82036451764347 +F1 1.01418002973925 3.0283600594785 +F2 1.02456128258909 3.04912256517818 +F3 -0.74363992043225 -0.48727984086450005 +F4 1.29823896534823 3.59647793069646 +F5 -1.12965055281585 -1.2593011056317 +F6 -0.522401797448688 -0.04480359489737595 +F7 0.327560711072239 1.6551214221444779 +F8 -1.3738693465664802 -1.7477386931329604 +F9 -0.7847891526325621 -0.5695783052651242 +F10 -0.280063201878434 0.43987359624313205 +F11 -0.251269847578052 0.49746030484389603 +F12 1.24602780723028 3.49205561446056 +F13 0.665734866338239 2.331469732676478 +F14 -0.889042985114811 -0.7780859702296219 +F15 -0.811956802515126 -0.6239136050302521 +F16 0.11591764582945 1.2318352916589 +F17 -0.156195990858492 0.687608018283016 +F18 -0.965770633683909 -0.9315412673678181 +F19 0.8035240337800391 2.6070480675600782 +F20 0.680306950765235 2.3606139015304697 +F21 -0.688388077896823 -0.376776155793646 +F22 0.7949653982850671 2.589930796570134 +F23 -1.11912925367142 -1.2382585073428398 +F24 -1.10593563527746 -1.2118712705549202 +F25 0.678934047810573 2.357868095621146 +F26 -0.937189288219405 -0.8743785764388099 +F27 -0.99973019311648 -0.9994603862329601 +F28 -0.31799033232181 0.36401933535638 +F29 -0.66141741897569 -0.3228348379513799 +F30 0.550511528315366 2.1010230566307317 +F31 -0.9325448302932858 -0.8650896605865717 +F32 -0.34253881283785104 0.3149223743242979 +F33 1.52951044529528 4.05902089059056 +F34 -0.0322228434081009 0.9355543131837982 +F35 -1.3936520437793 -1.7873040875586002 +F36 0.8687222580720692 2.737444516144138 +F37 0.6823592821191031 2.3647185642382063 +F38 1.37480393072457 3.74960786144914 +F39 0.239227247696375 1.47845449539275 +F40 -0.95692625177732 -0.9138525035546401 +F41 1.00830368417208 3.01660736834416 +F42 0.8288761564573699 2.65775231291474 +F43 -1.3476440812435702 -1.6952881624871403 +F44 0.529025296092455 2.0580505921849097 +F45 -0.340145172805159 0.319709654389682 +F46 0.964756233144604 2.929512466289208 +F47 -0.7310281367702399 -0.46205627354047984 +F48 0.5383478597420129 2.076695719484026 +F49 0.29953220850051193 1.5990644170010238 +F50 1.00798009601963 3.01596019203926 +F51 -0.0132115389102393 0.9735769221795214 +F52 -0.278915712970544 0.44216857405891197 +F53 0.06277454002648 1.12554908005296 +F54 0.3516746428630449 1.7033492857260897 +F55 0.137951639939762 1.275903279879524 +F56 1.33461560518154 3.66923121036308 +F57 0.43628478179779 1.87256956359558 +F58 1.11453590606852 3.22907181213704 +F59 1.29860068665808 3.59720137331616 +F60 0.0955549779502926 1.191109955900585 +F61 0.495998910273253 1.9919978205465059 +F62 -0.7833488766142149 -0.5666977532284299 +F63 -0.18096581167667 0.6380683766466599 +F64 1.30235947734143 3.60471895468286 +F65 0.280013344538779 1.560026689077558 +F66 1.30046727244606 3.60093454489212 +F67 -1.06463133806824 -1.12926267613648 +F68 -1.25938109004015 -1.5187621800803002 +F69 0.264024004173156 1.5280480083463122 +F70 -1.2529907420368602 -1.5059814840737205 +F71 1.4937353010847 3.9874706021694 +F72 0.970455224810024 2.940910449620048 +F73 -1.23960496737832 -1.47920993475664 +F74 -0.203565231196184 0.5928695376076321 +F75 -0.450539610910928 0.09892077817814404 +F76 0.411462370457529 1.822924740915058 +F77 -0.309713881057648 0.38057223788470396 +F78 0.9140794729087808 2.8281589458175613 +F79 1.31441543849981 3.62883087699962 +F80 -0.387324391709564 0.22535121658087198 +F81 -1.2144940378668605 -1.428988075733721 +F82 0.981754295455261 2.9635085909105223 +F83 0.0289330595561297 1.0578661191122594 +F84 1.24311989408419 3.48623978816838 +F85 -0.24182271418096696 0.5163545716380661 +F86 0.76870209333065 2.5374041866613 +F87 0.238515379111686 1.477030758223372 +F88 -0.338651978123051 0.32269604375389804 +F89 -0.4482656135303739 0.10346877293925216 +F90 0.5279056466564099 2.0558112933128196 +F91 1.37807956999098 3.75615913998196 +F92 1.04310859627148 3.08621719254296 +F93 -0.117482004574065 0.76503599085187 +F94 0.6232413973258649 2.2464827946517296 +F95 0.00403130502610116 1.0080626100522023 +F96 -1.34598938813891 -1.69197877627782 +F97 0.4549706581338591 1.9099413162677181 +F98 1.0192013773894302 3.0384027547788603 +F99 -0.291756302796704 0.416487394406592 +F100 -1.12688813864611 -1.25377627729222 +F101 0.557769384655089 2.115538769310178 +F102 1.1975908860910902 3.3951817721821804 +F103 1.15355341444875 3.3071068288975 +F104 0.946531581472272 2.893063162944544 +F105 1.22600261259885 3.4520052251977 +F106 1.0261210015913198 3.0522420031826396 +F107 0.841857769084618 2.6837155381692357 +F108 -0.4019324938447151 0.19613501231056985 +F109 0.19764363305971105 1.3952872661194222 +F110 -0.149480290579409 0.7010394188411819 +F111 0.6408321521409511 2.281664304281902 +F112 -0.966639287078902 -0.933278574157804 +F113 1.16738708670172 3.33477417340344 +F114 0.174326207230448 1.348652414460896 +F115 -0.5695094365752371 -0.13901887315047423 +F116 1.20806940422198 3.41613880844396 +F117 -1.24543884840423 -1.49087769680846 +F118 -1.15412531725971 -1.3082506345194198 +F119 -0.8296471214736691 -0.6592942429473383 +F120 0.736529671228746 2.473059342457492 +F121 -0.660047869199522 -0.3200957383990439 +F122 0.904640201633404 2.8092804032668077 +F123 -0.5986532323341109 -0.19730646466822188 +F124 0.691079988752513 2.382159977505026 +F125 -0.562592354003712 -0.12518470800742398 +F126 1.01123956644237 3.02247913288474 +F127 -0.41423096856367103 0.17153806287265794 +F128 0.619238453979786 2.238476907959572 +F129 -0.620591765205541 -0.2411835304110821 +F130 -1.27255930040442 -1.54511860080884 +F131 -0.3512359129578449 0.2975281740843102 +F132 1.00467461529444 3.00934923058888 +F133 -1.13541106483259 -1.2708221296651798 +F134 -1.26161085240342 -1.52322170480684 +F135 -0.38659519790583097 0.22680960418833807 +F136 0.872749204870088 2.745498409740176 +F137 -0.810751128748022 -0.621502257496044 +F138 -0.8197848282048691 -0.6395696564097382 +F139 0.20047162644758895 1.400943252895178 +F140 -0.604549610882948 -0.20909922176589602 +F141 0.228624680177828 1.457249360355656 +F142 1.28609633871252 3.57219267742504 +F143 -0.1679362832488 0.6641274335024 +F144 -0.257647674436839 0.484704651126322 +F145 0.74101019223122 2.48202038446244 +F146 0.132687409665163 1.265374819330326 +F147 -1.09538523785539 -1.1907704757107802 +F148 -0.4520741143592221 0.09585177128155575 +F149 -1.1796888761929298 -1.3593777523858597 +F150 1.07271995122987 3.14543990245974 +F151 -0.219846098067332 0.560307803865336 +F152 0.529493500001223 2.058987000002446 +F153 0.6270798132806121 2.2541596265612243 +F154 -1.01527420105315 -1.0305484021062998 +F155 0.746112185723826 2.492224371447652 +F156 -0.686265054564927 -0.37253010912985407 +F157 -1.3326095821448702 -1.6652191642897405 +F158 -0.3777879682136721 0.24442406357265578 +F159 -1.13401841717297 -1.2680368343459398 +F160 -0.296439244074386 0.407121511851228 +F161 0.40847742352928296 1.816954847058566 +F162 1.27982997837018 3.55965995674036 +F163 -0.7134413622708241 -0.4268827245416482 +F164 -0.349087095579928 0.301825808840144 +F165 0.6163069356830501 2.2326138713661 +F166 0.217564847263855 1.4351296945277099 +F167 -0.242902200286938 0.514195599426124 +F168 -1.01430232660254 -1.0286046532050799 +F169 -0.919646912875392 -0.839293825750784 +F170 -0.19937212012607394 0.6012557597478521 +F171 -1.31129333739411 -1.62258667478822 +F172 -0.721872176785536 -0.4437443535710719 +F173 -1.12678232313294 -1.2535646462658798 +F174 1.0904935742820798 3.1809871485641597 +F175 -1.17644925652159 -1.35289851304318 +F176 1.25195089868697 3.50390179737394 +F177 -0.5994841508457149 -0.19896830169142987 +F178 -1.12642775074296 -1.2528555014859202 +F179 -0.927361070785748 -0.8547221415714961 +F180 0.852558007271541 2.7051160145430817 +F181 0.740604316851619 2.481208633703238 +F182 -0.504950438684657 -0.00990087736931411 +F183 0.358858461465484 1.717716922930968 +F184 -0.0652057793033983 0.8695884413932033 +F185 -0.210512467576898 0.578975064846204 +F186 0.15407114587584306 1.308142291751686 +F187 0.7501398636830259 2.5002797273660518 +F188 -1.38405071916205 -1.7681014383240998 +F189 -1.3863784044176497 -1.7727568088352994 +F190 0.35465073693029203 1.709301473860584 +F191 -0.0527678764658988 0.8944642470682024 +F192 1.04452771030868 3.08905542061736 +F193 -1.0105215476872 -1.0210430953744 +F194 -0.6098963926022061 -0.21979278520441214 +F195 1.24019945405587 3.48039890811174 +F196 -1.2486190178628 -1.4972380357256 +F197 0.28873133501175297 1.577462670023506 +F198 -0.9444328759187892 -0.8888657518375784 +F199 -0.308547237330663 0.38290552533867395 diff --git a/q2_types/feature_data/tests/test_format.py b/q2_types/feature_data/tests/test_format.py index 548af8f0..e8d29a36 100644 --- a/q2_types/feature_data/tests/test_format.py +++ b/q2_types/feature_data/tests/test_format.py @@ -214,6 +214,46 @@ def test_differential_format(self): format.validate() + def test_differential_format_empty(self): + filepath = self.get_data_path('empty_differential.tsv') + temp_dir = self.temp_dir.name + shutil.copy(filepath, + os.path.join(temp_dir, 'empty_differential.tsv')) + format = DifferentialDirectoryFormat(temp_dir, mode='r') + + with self.assertRaisesRegex(ValidationError, 'Differential'): + format.validate() + + def test_differential_format_bad(self): + filepath = self.get_data_path('bad_differential.tsv') + temp_dir = self.temp_dir.name + shutil.copy(filepath, + os.path.join(temp_dir, 'bad_differential.tsv')) + format = DifferentialDirectoryFormat(temp_dir, mode='r') + + with self.assertRaisesRegex(ValidationError, 'Differential'): + format.validate() + def test_differential_format_inf(self): + filepath = self.get_data_path('inf_differential.tsv') + temp_dir = self.temp_dir.name + shutil.copy(filepath, + os.path.join(temp_dir, 'inf_differential.tsv')) + format = DifferentialDirectoryFormat(temp_dir, mode='r') + + with self.assertRaisesRegex(ValidationError, 'Differential'): + format.validate() + + def test_differential_format_inf(self): + filepath = self.get_data_path('nan_differential.tsv') + temp_dir = self.temp_dir.name + shutil.copy(filepath, + os.path.join(temp_dir, 'nan_differential.tsv')) + format = DifferentialDirectoryFormat(temp_dir, mode='r') + + with self.assertRaisesRegex(ValidationError, 'Differential'): + format.validate() + + if __name__ == '__main__': unittest.main() From e8a2298abc949b6fb4a75032df99cefe18fccca6 Mon Sep 17 00:00:00 2001 From: mortonjt Date: Fri, 24 May 2019 12:34:26 -0400 Subject: [PATCH 09/15] TST: adding in nan differential --- .../tests/data/nan_differential.tsv | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 q2_types/feature_data/tests/data/nan_differential.tsv diff --git a/q2_types/feature_data/tests/data/nan_differential.tsv b/q2_types/feature_data/tests/data/nan_differential.tsv new file mode 100644 index 00000000..48385f9d --- /dev/null +++ b/q2_types/feature_data/tests/data/nan_differential.tsv @@ -0,0 +1,201 @@ +featureid effect +F0 +F1 1.01418002973925 +F2 1.02456128258909 +F3 -0.74363992043225 +F4 1.29823896534823 +F5 -1.12965055281585 +F6 -0.522401797448688 +F7 0.327560711072239 +F8 -1.3738693465664802 +F9 -0.7847891526325621 +F10 -0.280063201878434 +F11 -0.251269847578052 +F12 1.24602780723028 +F13 0.665734866338239 +F14 -0.889042985114811 +F15 -0.811956802515126 +F16 0.11591764582945 +F17 -0.156195990858492 +F18 -0.965770633683909 +F19 0.8035240337800391 +F20 0.680306950765235 +F21 -0.688388077896823 +F22 0.7949653982850671 +F23 -1.11912925367142 +F24 -1.10593563527746 +F25 0.678934047810573 +F26 -0.937189288219405 +F27 -0.99973019311648 +F28 -0.31799033232181 +F29 -0.66141741897569 +F30 0.550511528315366 +F31 -0.9325448302932858 +F32 -0.34253881283785104 +F33 1.52951044529528 +F34 -0.0322228434081009 +F35 -1.3936520437793 +F36 0.8687222580720692 +F37 0.6823592821191031 +F38 1.37480393072457 +F39 0.239227247696375 +F40 -0.95692625177732 +F41 1.00830368417208 +F42 0.8288761564573699 +F43 -1.3476440812435702 +F44 0.529025296092455 +F45 -0.340145172805159 +F46 0.964756233144604 +F47 -0.7310281367702399 +F48 0.5383478597420129 +F49 0.29953220850051193 +F50 1.00798009601963 +F51 -0.0132115389102393 +F52 -0.278915712970544 +F53 0.06277454002648 +F54 0.3516746428630449 +F55 0.137951639939762 +F56 1.33461560518154 +F57 0.43628478179779 +F58 1.11453590606852 +F59 1.29860068665808 +F60 0.0955549779502926 +F61 0.495998910273253 +F62 -0.7833488766142149 +F63 -0.18096581167667 +F64 1.30235947734143 +F65 0.280013344538779 +F66 1.30046727244606 +F67 -1.06463133806824 +F68 -1.25938109004015 +F69 0.264024004173156 +F70 -1.2529907420368602 +F71 1.4937353010847 +F72 0.970455224810024 +F73 -1.23960496737832 +F74 -0.203565231196184 +F75 -0.450539610910928 +F76 0.411462370457529 +F77 -0.309713881057648 +F78 0.9140794729087808 +F79 1.31441543849981 +F80 -0.387324391709564 +F81 -1.2144940378668605 +F82 0.981754295455261 +F83 0.0289330595561297 +F84 1.24311989408419 +F85 -0.24182271418096696 +F86 0.76870209333065 +F87 0.238515379111686 +F88 -0.338651978123051 +F89 -0.4482656135303739 +F90 0.5279056466564099 +F91 1.37807956999098 +F92 1.04310859627148 +F93 -0.117482004574065 +F94 0.6232413973258649 +F95 0.00403130502610116 +F96 -1.34598938813891 +F97 0.4549706581338591 +F98 1.0192013773894302 +F99 -0.291756302796704 +F100 -1.12688813864611 +F101 0.557769384655089 +F102 1.1975908860910902 +F103 1.15355341444875 +F104 0.946531581472272 +F105 1.22600261259885 +F106 1.0261210015913198 +F107 0.841857769084618 +F108 -0.4019324938447151 +F109 0.19764363305971105 +F110 -0.149480290579409 +F111 0.6408321521409511 +F112 -0.966639287078902 +F113 1.16738708670172 +F114 0.174326207230448 +F115 -0.5695094365752371 +F116 1.20806940422198 +F117 -1.24543884840423 +F118 -1.15412531725971 +F119 -0.8296471214736691 +F120 0.736529671228746 +F121 -0.660047869199522 +F122 0.904640201633404 +F123 -0.5986532323341109 +F124 0.691079988752513 +F125 -0.562592354003712 +F126 1.01123956644237 +F127 -0.41423096856367103 +F128 0.619238453979786 +F129 -0.620591765205541 +F130 -1.27255930040442 +F131 -0.3512359129578449 +F132 1.00467461529444 +F133 -1.13541106483259 +F134 -1.26161085240342 +F135 -0.38659519790583097 +F136 0.872749204870088 +F137 -0.810751128748022 +F138 -0.8197848282048691 +F139 0.20047162644758895 +F140 -0.604549610882948 +F141 0.228624680177828 +F142 1.28609633871252 +F143 -0.1679362832488 +F144 -0.257647674436839 +F145 0.74101019223122 +F146 0.132687409665163 +F147 -1.09538523785539 +F148 -0.4520741143592221 +F149 -1.1796888761929298 +F150 1.07271995122987 +F151 -0.219846098067332 +F152 0.529493500001223 +F153 0.6270798132806121 +F154 -1.01527420105315 +F155 0.746112185723826 +F156 -0.686265054564927 +F157 -1.3326095821448702 +F158 -0.3777879682136721 +F159 -1.13401841717297 +F160 -0.296439244074386 +F161 0.40847742352928296 +F162 1.27982997837018 +F163 -0.7134413622708241 +F164 -0.349087095579928 +F165 0.6163069356830501 +F166 0.217564847263855 +F167 -0.242902200286938 +F168 -1.01430232660254 +F169 -0.919646912875392 +F170 -0.19937212012607394 +F171 -1.31129333739411 +F172 -0.721872176785536 +F173 -1.12678232313294 +F174 1.0904935742820798 +F175 -1.17644925652159 +F176 1.25195089868697 +F177 -0.5994841508457149 +F178 -1.12642775074296 +F179 -0.927361070785748 +F180 0.852558007271541 +F181 0.740604316851619 +F182 -0.504950438684657 +F183 0.358858461465484 +F184 -0.0652057793033983 +F185 -0.210512467576898 +F186 0.15407114587584306 +F187 0.7501398636830259 +F188 -1.38405071916205 +F189 -1.3863784044176497 +F190 0.35465073693029203 +F191 -0.0527678764658988 +F192 1.04452771030868 +F193 -1.0105215476872 +F194 -0.6098963926022061 +F195 1.24019945405587 +F196 -1.2486190178628 +F197 0.28873133501175297 +F198 -0.9444328759187892 +F199 -0.308547237330663 From 1233f9934e085b52f2d0b4167c7097eacb2eeb1d Mon Sep 17 00:00:00 2001 From: mortonjt Date: Fri, 24 May 2019 12:51:35 -0400 Subject: [PATCH 10/15] TST: flake8 and fix nan test. Now let's pray --- q2_types/feature_data/tests/test_format.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/q2_types/feature_data/tests/test_format.py b/q2_types/feature_data/tests/test_format.py index e8d29a36..0894af07 100644 --- a/q2_types/feature_data/tests/test_format.py +++ b/q2_types/feature_data/tests/test_format.py @@ -220,7 +220,7 @@ def test_differential_format_empty(self): shutil.copy(filepath, os.path.join(temp_dir, 'empty_differential.tsv')) format = DifferentialDirectoryFormat(temp_dir, mode='r') - + with self.assertRaisesRegex(ValidationError, 'Differential'): format.validate() @@ -230,8 +230,8 @@ def test_differential_format_bad(self): shutil.copy(filepath, os.path.join(temp_dir, 'bad_differential.tsv')) format = DifferentialDirectoryFormat(temp_dir, mode='r') - - with self.assertRaisesRegex(ValidationError, 'Differential'): + + with self.assertRaisesRegex(ValidationError, 'Differential'): format.validate() def test_differential_format_inf(self): @@ -240,20 +240,20 @@ def test_differential_format_inf(self): shutil.copy(filepath, os.path.join(temp_dir, 'inf_differential.tsv')) format = DifferentialDirectoryFormat(temp_dir, mode='r') - - with self.assertRaisesRegex(ValidationError, 'Differential'): + + with self.assertRaisesRegex(ValidationError, 'Differential'): format.validate() - def test_differential_format_inf(self): + def test_differential_format_nan(self): filepath = self.get_data_path('nan_differential.tsv') temp_dir = self.temp_dir.name shutil.copy(filepath, os.path.join(temp_dir, 'nan_differential.tsv')) format = DifferentialDirectoryFormat(temp_dir, mode='r') - - with self.assertRaisesRegex(ValidationError, 'Differential'): + + with self.assertRaisesRegex(ValidationError, 'Differential'): format.validate() - - + + if __name__ == '__main__': unittest.main() From 179771ea8c36a97f9fed9b5079c97d71308aa9bb Mon Sep 17 00:00:00 2001 From: mortonjt Date: Mon, 24 May 2021 11:31:12 -0600 Subject: [PATCH 11/15] adding monte carlo type --- q2_types/feature_data/__init__.py | 12 ++++--- q2_types/feature_data/_format.py | 29 +++++++++++---- q2_types/feature_data/_transformer.py | 15 ++++++-- q2_types/feature_data/_type.py | 10 ++++-- .../tests/data/monte-carlo-samples.az | 0 q2_types/feature_data/tests/test_format.py | 36 +++++++++++++++++-- 6 files changed, 86 insertions(+), 16 deletions(-) create mode 100644 q2_types/feature_data/tests/data/monte-carlo-samples.az diff --git a/q2_types/feature_data/__init__.py b/q2_types/feature_data/__init__.py index c29cf1ba..0ed11f1c 100644 --- a/q2_types/feature_data/__init__.py +++ b/q2_types/feature_data/__init__.py @@ -13,11 +13,13 @@ HeaderlessTSVTaxonomyDirectoryFormat, TSVTaxonomyFormat, TSVTaxonomyDirectoryFormat, DNAFASTAFormat, DNASequencesDirectoryFormat, PairedDNASequencesDirectoryFormat, AlignedDNAFASTAFormat, - AlignedDNASequencesDirectoryFormat, DifferentialFormat, - DifferentialDirectoryFormat) + AlignedDNASequencesDirectoryFormat, + DifferentialFormat, DifferentialDirectoryFormat, + MonteCarloTensorFormat, MonteCarloTensorDirectoryFormat +) from ._type import ( FeatureData, Taxonomy, Sequence, PairedEndSequence, AlignedSequence, - Differential) + Differential, MonteCarloTensor) # TODO remove these imports when tests are rewritten. Remove from __all__ too from ._transformer import DNAIterator, PairedDNAIterator, AlignedDNAIterator @@ -30,6 +32,8 @@ 'AlignedDNAFASTAFormat', 'AlignedDNASequencesDirectoryFormat', 'FeatureData', 'Taxonomy', 'Sequence', 'PairedEndSequence', 'AlignedSequence', 'DNAIterator', 'PairedDNAIterator', - 'AlignedDNAIterator', 'Differential', 'DifferentialDirectoryFormat'] + 'AlignedDNAIterator', 'Differential', 'DifferentialDirectoryFormat', + 'MonteCarloTensor', 'MonteCarloTensorDirectoryFormat' +] importlib.import_module('q2_types.feature_data._transformer') diff --git a/q2_types/feature_data/_format.py b/q2_types/feature_data/_format.py index d14bc94b..a12d8ff6 100644 --- a/q2_types/feature_data/_format.py +++ b/q2_types/feature_data/_format.py @@ -11,7 +11,7 @@ import qiime2 import qiime2.plugin.model as model from qiime2.plugin import ValidationError - +import arviz as az from ..plugin_setup import plugin @@ -203,7 +203,7 @@ def validate(self, *args): 'at least 1 column') ) - + types = md.dtypes for t in types: if not np.issubdtype(t, np.number): @@ -212,23 +212,40 @@ def validate(self, *args): 'continuously valued quantities') ) - + for c in md.columns: if (np.isinf(md[c].values).sum() > 0 or pd.isnull(md[c]).sum() > 0): raise ValueError( ('Differential contains infs or nans.') ) - - + + DifferentialDirectoryFormat = model.SingleFileDirectoryFormat( 'DifferentialDirectoryFormat', 'differentials.tsv', DifferentialFormat) + +class MonteCarloTensorFormat(model.BinaryFileFormat): + + def sniff(self): + try: + az.InferenceData.from_netcdf(str(self)) + return True + except Exception: + return False + + +MonteCarloTensorDirectoryFormat = model.SingleFileDirectoryFormat( + 'MonteCarloTensorDirectoryFormat', 'monte-carlo-samples.az', + MonteCarloTensorFormat) + + plugin.register_formats( TSVTaxonomyFormat, TSVTaxonomyDirectoryFormat, HeaderlessTSVTaxonomyFormat, HeaderlessTSVTaxonomyDirectoryFormat, TaxonomyFormat, TaxonomyDirectoryFormat, DNAFASTAFormat, DNASequencesDirectoryFormat, PairedDNASequencesDirectoryFormat, AlignedDNAFASTAFormat, AlignedDNASequencesDirectoryFormat, - DifferentialFormat, DifferentialDirectoryFormat + DifferentialFormat, DifferentialDirectoryFormat, + MonteCarloTensorFormat, MonteCarloTensorDirectoryFormat ) diff --git a/q2_types/feature_data/_transformer.py b/q2_types/feature_data/_transformer.py index b1066153..b3e24060 100644 --- a/q2_types/feature_data/_transformer.py +++ b/q2_types/feature_data/_transformer.py @@ -13,12 +13,13 @@ import biom import skbio import qiime2 - +import arviz as az from ..plugin_setup import plugin from ..feature_table import BIOMV210Format from . import (TaxonomyFormat, HeaderlessTSVTaxonomyFormat, TSVTaxonomyFormat, DNAFASTAFormat, PairedDNASequencesDirectoryFormat, - AlignedDNAFASTAFormat, DifferentialFormat) + AlignedDNAFASTAFormat, DifferentialFormat, + MonteCarloTensorFormat) # Taxonomy format transformers @@ -375,3 +376,13 @@ def _224(df: pd.DataFrame) -> DifferentialFormat: ff = DifferentialFormat() qiime2.Metadata(data).save(str(ff)) return ff + +@plugin.register_transformer +def _225(ff: MonteCarloTensorFormat) -> az.InferenceData: + return az.InferenceData.from_netcdf(str(ff)) + +@plugin.register_transformer +def _226(obj: az.InferenceData) -> MonteCarloTensorFormat: + ff = MonteCarloTensorFormat() + obj.to_netcdf(str(ff)) + return ff diff --git a/q2_types/feature_data/_type.py b/q2_types/feature_data/_type.py index 147136e3..ab9f94cc 100644 --- a/q2_types/feature_data/_type.py +++ b/q2_types/feature_data/_type.py @@ -12,7 +12,8 @@ from . import (TSVTaxonomyDirectoryFormat, DNASequencesDirectoryFormat, PairedDNASequencesDirectoryFormat, AlignedDNASequencesDirectoryFormat, - DifferentialDirectoryFormat) + DifferentialDirectoryFormat, + MonteCarloTensorDirectoryFormat) FeatureData = SemanticType('FeatureData', field_names='type') @@ -30,9 +31,12 @@ Differential = SemanticType('Differential', variant_of=FeatureData.field['type']) +MonteCarloTensor = SemanticType('MonteCarloTensor') + + plugin.register_semantic_types(FeatureData, Taxonomy, Sequence, PairedEndSequence, AlignedSequence, - Differential) + Differential, MonteCarloTensor) plugin.register_semantic_type_to_format( @@ -49,3 +53,5 @@ artifact_format=AlignedDNASequencesDirectoryFormat) plugin.register_semantic_type_to_format( FeatureData[Differential], DifferentialDirectoryFormat) +plugin.register_semantic_type_to_format( + MonteCarloTensor, MonteCarloTensorDirectoryFormat) diff --git a/q2_types/feature_data/tests/data/monte-carlo-samples.az b/q2_types/feature_data/tests/data/monte-carlo-samples.az new file mode 100644 index 00000000..e69de29b diff --git a/q2_types/feature_data/tests/test_format.py b/q2_types/feature_data/tests/test_format.py index 0894af07..4b6dcd2d 100644 --- a/q2_types/feature_data/tests/test_format.py +++ b/q2_types/feature_data/tests/test_format.py @@ -16,10 +16,13 @@ HeaderlessTSVTaxonomyDirectoryFormat, TSVTaxonomyFormat, TSVTaxonomyDirectoryFormat, DNAFASTAFormat, DNASequencesDirectoryFormat, PairedDNASequencesDirectoryFormat, AlignedDNAFASTAFormat, - AlignedDNASequencesDirectoryFormat, DifferentialDirectoryFormat + AlignedDNASequencesDirectoryFormat, DifferentialDirectoryFormat, + MonteCarloTensorDirectoryFormat ) from qiime2.plugin.testing import TestPluginBase from qiime2.plugin import ValidationError +import arviz as az +import numpy as np class TestTaxonomyFormats(TestPluginBase): @@ -33,7 +36,6 @@ def test_taxonomy_format_validate_positive(self): for filepath in filepaths: format = TaxonomyFormat(filepath, mode='r') - format.validate() def test_taxonomy_format_validate_negative(self): @@ -255,5 +257,35 @@ def test_differential_format_nan(self): format.validate() +class TestMonteCarloTensorFormat(TestPluginBase): + + package = 'q2_types.feature_data.tests' + + def test_monte_carlo_format(self): + # Note that this file is empty, we are using it as a + # placeholder to place the az Inference object + filepath = self.get_data_path('monte-carlo-samples.az') + + size = 100 + dataset = az.convert_to_inference_data(np.random.randn(size)) + dataset.to_netcdf(filepath) + + temp_dir = self.temp_dir.name + shutil.copy(filepath, + os.path.join(temp_dir, 'monte-carlo-samples.az')) + format = MonteCarloTensorDirectoryFormat(temp_dir, mode='r') + format.validate() + + + def test_monte_carlo_format_bad(self): + filepath = self.get_data_path('nan_differential.tsv') + temp_dir = self.temp_dir.name + shutil.copy(filepath, + os.path.join(temp_dir, 'nan_differential.tsv')) + format = MonteCarloTensorDirectoryFormat(temp_dir, mode='r') + with self.assertRaisesRegex(ValidationError, 'MonteCarloTensor'): + format.validate() + + if __name__ == '__main__': unittest.main() From 4b1cb85441f2c2f63aaf668a2d21d5dad5cc1764 Mon Sep 17 00:00:00 2001 From: mortonjt Date: Mon, 24 May 2021 11:48:43 -0600 Subject: [PATCH 12/15] update --- q2_types/feature_data/_transformer.py | 6 +- q2_types/feature_data/_type.py | 16 ++---- q2_types/feature_data/tests/test_format.py | 56 +------------------ .../feature_data/tests/test_transformer.py | 37 ------------ 4 files changed, 10 insertions(+), 105 deletions(-) diff --git a/q2_types/feature_data/_transformer.py b/q2_types/feature_data/_transformer.py index 445ffeba..06579771 100644 --- a/q2_types/feature_data/_transformer.py +++ b/q2_types/feature_data/_transformer.py @@ -18,7 +18,7 @@ from ..feature_table import BIOMV210Format from . import (TaxonomyFormat, HeaderlessTSVTaxonomyFormat, TSVTaxonomyFormat, DNAFASTAFormat, PairedDNASequencesDirectoryFormat, - MonteCarloTensorFormat, MonteCarloTensorDirectoryFormat + MonteCarloTensorFormat, AlignedDNAFASTAFormat, DifferentialFormat, ProteinFASTAFormat, AlignedProteinFASTAFormat, RNAFASTAFormat, AlignedRNAFASTAFormat, PairedRNASequencesDirectoryFormat @@ -634,20 +634,24 @@ def _64(data: PairedRNAIterator) -> PairedRNASequencesDirectoryFormat: def _222(ff: DifferentialFormat) -> pd.DataFrame: return qiime2.Metadata.load(str(ff)).to_dataframe() + @plugin.register_transformer def _223(ff: DifferentialFormat) -> qiime2.Metadata: return qiime2.Metadata.load(str(ff)) + @plugin.register_transformer def _224(data: pd.DataFrame) -> DifferentialFormat: ff = DifferentialFormat() qiime2.Metadata(data).save(str(ff)) return ff + @plugin.register_transformer def _225(ff: MonteCarloTensorFormat) -> az.InferenceData: return az.InferenceData.from_netcdf(str(ff)) + @plugin.register_transformer def _226(obj: az.InferenceData) -> MonteCarloTensorFormat: ff = MonteCarloTensorFormat() diff --git a/q2_types/feature_data/_type.py b/q2_types/feature_data/_type.py index 14f667ce..6bb5e7b1 100644 --- a/q2_types/feature_data/_type.py +++ b/q2_types/feature_data/_type.py @@ -12,7 +12,7 @@ from . import (TSVTaxonomyDirectoryFormat, DNASequencesDirectoryFormat, PairedDNASequencesDirectoryFormat, AlignedDNASequencesDirectoryFormat, - MonteCarloTensorFormat, MonteCarloTensorDirectoryFormat, + MonteCarloTensorDirectoryFormat, DifferentialDirectoryFormat, ProteinSequencesDirectoryFormat, AlignedProteinSequencesDirectoryFormat, RNASequencesDirectoryFormat, AlignedRNASequencesDirectoryFormat, @@ -36,17 +36,9 @@ AlignedSequence = SemanticType('AlignedSequence', variant_of=FeatureData.field['type']) -<<<<<<< HEAD -Differential = SemanticType('Differential', - variant_of=FeatureData.field['type']) - MonteCarloTensor = SemanticType('MonteCarloTensor') -plugin.register_semantic_types(FeatureData, Taxonomy, Sequence, - PairedEndSequence, AlignedSequence, - Differential, MonteCarloTensor) -======= AlignedRNASequence = SemanticType('AlignedRNASequence', variant_of=FeatureData.field['type']) @@ -61,10 +53,10 @@ plugin.register_semantic_types(FeatureData, Taxonomy, Sequence, PairedEndSequence, AlignedSequence, - Differential, ProteinSequence, + Differential, MonteCarloTensor, + ProteinSequence, AlignedProteinSequence, RNASequence, AlignedRNASequence, PairedEndRNASequence) ->>>>>>> 623c9aad86e777c1029ff286697e7d532a101258 plugin.register_semantic_type_to_format( @@ -89,6 +81,7 @@ FeatureData[Differential], DifferentialDirectoryFormat) plugin.register_semantic_type_to_format( MonteCarloTensor, MonteCarloTensorDirectoryFormat) +plugin.register_semantic_type_to_format( FeatureData[AlignedRNASequence], artifact_format=AlignedRNASequencesDirectoryFormat) plugin.register_semantic_type_to_format( @@ -99,4 +92,3 @@ plugin.register_semantic_type_to_format( FeatureData[AlignedProteinSequence], artifact_format=AlignedProteinSequencesDirectoryFormat) ->>>>>>> 623c9aad86e777c1029ff286697e7d532a101258 diff --git a/q2_types/feature_data/tests/test_format.py b/q2_types/feature_data/tests/test_format.py index e0228473..fbe35b92 100644 --- a/q2_types/feature_data/tests/test_format.py +++ b/q2_types/feature_data/tests/test_format.py @@ -21,7 +21,7 @@ AlignedProteinSequencesDirectoryFormat, ProteinSequencesDirectoryFormat, RNAFASTAFormat, RNASequencesDirectoryFormat, AlignedRNAFASTAFormat, AlignedRNASequencesDirectoryFormat, - MonteCarloTensor, MonteCarloTensorDirectoryFormat + MonteCarloTensorDirectoryFormat ) from qiime2.plugin.testing import TestPluginBase from qiime2.plugin import ValidationError @@ -539,59 +539,6 @@ def test_aligned_protein_sequences_directory_format(self): format.validate() -class TestDifferentialFormat(TestPluginBase): - package = 'q2_types.feature_data.tests' - - def test_differential_format(self): - filepath = self.get_data_path('differentials.tsv') - temp_dir = self.temp_dir.name - shutil.copy(filepath, - os.path.join(temp_dir, 'differentials.tsv')) - format = DifferentialDirectoryFormat(temp_dir, mode='r') - - format.validate() - - def test_differential_format_empty(self): - filepath = self.get_data_path('empty_differential.tsv') - temp_dir = self.temp_dir.name - shutil.copy(filepath, - os.path.join(temp_dir, 'empty_differential.tsv')) - format = DifferentialDirectoryFormat(temp_dir, mode='r') - - with self.assertRaisesRegex(ValidationError, 'Differential'): - format.validate() - - def test_differential_format_bad(self): - filepath = self.get_data_path('bad_differential.tsv') - temp_dir = self.temp_dir.name - shutil.copy(filepath, - os.path.join(temp_dir, 'bad_differential.tsv')) - format = DifferentialDirectoryFormat(temp_dir, mode='r') - - with self.assertRaisesRegex(ValidationError, 'Differential'): - format.validate() - - def test_differential_format_inf(self): - filepath = self.get_data_path('inf_differential.tsv') - temp_dir = self.temp_dir.name - shutil.copy(filepath, - os.path.join(temp_dir, 'inf_differential.tsv')) - format = DifferentialDirectoryFormat(temp_dir, mode='r') - - with self.assertRaisesRegex(ValidationError, 'Differential'): - format.validate() - - def test_differential_format_nan(self): - filepath = self.get_data_path('nan_differential.tsv') - temp_dir = self.temp_dir.name - shutil.copy(filepath, - os.path.join(temp_dir, 'nan_differential.tsv')) - format = DifferentialDirectoryFormat(temp_dir, mode='r') - - with self.assertRaisesRegex(ValidationError, 'Differential'): - format.validate() - - class TestMonteCarloTensorFormat(TestPluginBase): package = 'q2_types.feature_data.tests' @@ -611,7 +558,6 @@ def test_monte_carlo_format(self): format = MonteCarloTensorDirectoryFormat(temp_dir, mode='r') format.validate() - def test_monte_carlo_format_bad(self): filepath = self.get_data_path('nan_differential.tsv') temp_dir = self.temp_dir.name diff --git a/q2_types/feature_data/tests/test_transformer.py b/q2_types/feature_data/tests/test_transformer.py index 35df915e..6b8714c2 100644 --- a/q2_types/feature_data/tests/test_transformer.py +++ b/q2_types/feature_data/tests/test_transformer.py @@ -928,43 +928,6 @@ def test_aligned_rna_fasta_format_to_rna_iterator(self): self.assertEqual(observed, expected) -class TestDifferentialTransformer(TestPluginBase): - package = 'q2_types.feature_data.tests' - - def test_differential_to_df(self): - - _, obs = self.transform_format(DifferentialFormat, pd.DataFrame, - filename='differentials.tsv') - - # sniff to see if the first 4 feature ids are the same - exp = ['F0', 'F1', 'F2', 'F3'] - obs = list(obs.index[:4]) - self.assertListEqual(exp, obs) - - def test_differential_to_md(self): - - _, obs = self.transform_format(DifferentialFormat, qiime2.Metadata, - filename='differentials.tsv') - obs = obs.to_dataframe() - # sniff to see if the first 4 feature ids are the same - exp = ['F0', 'F1', 'F2', 'F3'] - obs = list(obs.index[:4]) - self.assertListEqual(exp, obs) - - def test_df_to_differential(self): - transformer = self.get_transformer(pd.DataFrame, DifferentialFormat) - - index = pd.Index(['SEQUENCE1', 'SEQUENCE2', 'SEQUENCE3']) - index.name = 'featureid' - input = pd.DataFrame( - [-1.3, 0.1, 1.2], index=index, columns=['differential'], - dtype=float) - - obs = transformer(input) - - self.assertIsInstance(obs, DifferentialFormat) - - class TestProteinFASTAFormatTransformers(TestPluginBase): package = 'q2_types.feature_data.tests' From bbae6758bf5749313641110fffc66d25a38eec6b Mon Sep 17 00:00:00 2001 From: mortonjt Date: Mon, 24 May 2021 12:00:10 -0600 Subject: [PATCH 13/15] fixing more merge issues --- q2_types/feature_data/_type.py | 2 - .../tests/data/bad_differential.tsv | 172 ------------------ .../tests/data/inf_differential.tsv | 2 +- .../feature_data/tests/test_transformer.py | 61 ++++--- 4 files changed, 38 insertions(+), 199 deletions(-) diff --git a/q2_types/feature_data/_type.py b/q2_types/feature_data/_type.py index 6bb5e7b1..2f07c17a 100644 --- a/q2_types/feature_data/_type.py +++ b/q2_types/feature_data/_type.py @@ -77,8 +77,6 @@ plugin.register_semantic_type_to_format( FeatureData[AlignedSequence], artifact_format=AlignedDNASequencesDirectoryFormat) -plugin.register_semantic_type_to_format( - FeatureData[Differential], DifferentialDirectoryFormat) plugin.register_semantic_type_to_format( MonteCarloTensor, MonteCarloTensorDirectoryFormat) plugin.register_semantic_type_to_format( diff --git a/q2_types/feature_data/tests/data/bad_differential.tsv b/q2_types/feature_data/tests/data/bad_differential.tsv index 0bdb955f..34eef936 100644 --- a/q2_types/feature_data/tests/data/bad_differential.tsv +++ b/q2_types/feature_data/tests/data/bad_differential.tsv @@ -30,175 +30,3 @@ F27 -0.9997301931164799 0 F28 -0.31799033232181 0 F29 -0.66141741897569 0 F30 0.550511528315366 0 -<<<<<<< HEAD -F31 -0.9325448302932858 0 -F32 -0.34253881283785104 0 -F33 1.52951044529528 0 -F34 -0.0322228434081009 0 -F35 -1.3936520437793 0 -F36 0.8687222580720692 0 -F37 0.6823592821191031 0 -F38 1.37480393072457 0 -F39 0.239227247696375 0 -F40 -0.9569262517773199 0 -F41 1.00830368417208 0 -F42 0.8288761564573699 0 -F43 -1.3476440812435702 0 -F44 0.529025296092455 0 -F45 -0.340145172805159 0 -F46 0.964756233144604 0 -F47 -0.7310281367702399 0 -F48 0.5383478597420129 0 -F49 0.29953220850051193 0 -F50 1.00798009601963 0 -F51 -0.0132115389102393 0 -F52 -0.27891571297054396 0 -F53 0.06277454002648 0 -F54 0.3516746428630449 0 -F55 0.137951639939762 0 -F56 1.33461560518154 0 -F57 0.43628478179779 0 -F58 1.11453590606852 0 -F59 1.2986006866580801 0 -F60 0.0955549779502926 0 -F61 0.49599891027325305 0 -F62 -0.7833488766142149 0 -F63 -0.18096581167667 0 -F64 1.30235947734143 0 -F65 0.280013344538779 0 -F66 1.30046727244606 0 -F67 -1.06463133806824 0 -F68 -1.2593810900401499 0 -F69 0.264024004173156 0 -F70 -1.2529907420368602 0 -F71 1.4937353010846999 0 -F72 0.970455224810024 0 -F73 -1.23960496737832 0 -F74 -0.20356523119618397 0 -F75 -0.450539610910928 0 -F76 0.411462370457529 0 -F77 -0.309713881057648 0 -F78 0.9140794729087808 0 -F79 1.3144154384998101 0 -F80 -0.38732439170956395 0 -F81 -1.2144940378668605 0 -F82 0.981754295455261 0 -F83 0.0289330595561297 0 -F84 1.2431198940841899 0 -F85 -0.24182271418096696 0 -F86 0.76870209333065 0 -F87 0.23851537911168602 0 -F88 -0.338651978123051 0 -F89 -0.4482656135303739 0 -F90 0.5279056466564099 0 -F91 1.37807956999098 0 -F92 1.0431085962714801 0 -F93 -0.117482004574065 0 -F94 0.6232413973258649 0 -F95 0.00403130502610116 0 -F96 -1.34598938813891 0 -F97 0.4549706581338591 0 -F98 1.0192013773894302 0 -F99 -0.291756302796704 0 -F100 -1.12688813864611 0 -F101 0.557769384655089 0 -F102 1.1975908860910902 0 -F103 1.15355341444875 0 -F104 0.946531581472272 0 -F105 1.2260026125988501 0 -F106 1.0261210015913198 0 -F107 0.841857769084618 0 -F108 -0.4019324938447151 0 -F109 0.19764363305971105 0 -F110 -0.149480290579409 0 -F111 0.6408321521409511 0 -F112 -0.966639287078902 0 -F113 1.16738708670172 0 -F114 0.174326207230448 0 -F115 -0.5695094365752371 0 -F116 1.20806940422198 0 -F117 -1.24543884840423 0 -F118 -1.15412531725971 0 -F119 -0.8296471214736691 0 -F120 0.736529671228746 0 -F121 -0.660047869199522 0 -F122 0.9046402016334041 0 -F123 -0.5986532323341109 0 -F124 0.691079988752513 0 -F125 -0.562592354003712 0 -F126 1.01123956644237 0 -F127 -0.41423096856367103 0 -F128 0.619238453979786 0 -F129 -0.620591765205541 0 -F130 -1.27255930040442 0 -F131 -0.3512359129578449 0 -F132 1.00467461529444 0 -F133 -1.1354110648325901 0 -F134 -1.26161085240342 0 -F135 -0.38659519790583097 0 -F136 0.872749204870088 0 -F137 -0.810751128748022 0 -F138 -0.8197848282048691 0 -F139 0.20047162644758892 0 -F140 -0.604549610882948 0 -F141 0.228624680177828 0 -F142 1.28609633871252 0 -F143 -0.1679362832488 0 -F144 -0.257647674436839 0 -F145 0.74101019223122 0 -F146 0.13268740966516301 0 -F147 -1.09538523785539 0 -F148 -0.4520741143592221 0 -F149 -1.1796888761929298 0 -F150 1.07271995122987 0 -F151 -0.219846098067332 0 -F152 0.529493500001223 0 -F153 0.6270798132806121 0 -F154 -1.01527420105315 0 -F155 0.746112185723826 0 -F156 -0.686265054564927 0 -F157 -1.3326095821448702 0 -F158 -0.3777879682136721 0 -F159 -1.13401841717297 0 -F160 -0.296439244074386 0 -F161 0.40847742352928296 0 -F162 1.27982997837018 0 -F163 -0.7134413622708241 0 -F164 -0.349087095579928 0 -F165 0.6163069356830501 0 -F166 0.217564847263855 0 -F167 -0.242902200286938 0 -F168 -1.01430232660254 0 -F169 -0.9196469128753921 0 -F170 -0.19937212012607394 0 -F171 -1.31129333739411 0 -F172 -0.721872176785536 0 -F173 -1.12678232313294 0 -F174 1.0904935742820798 0 -F175 -1.17644925652159 0 -F176 1.25195089868697 0 -F177 -0.5994841508457149 0 -F178 -1.1264277507429599 0 -F179 -0.9273610707857479 0 -F180 0.852558007271541 0 -F181 0.740604316851619 0 -F182 -0.504950438684657 0 -F183 0.358858461465484 0 -F184 -0.0652057793033983 0 -F185 -0.210512467576898 0 -F186 0.15407114587584306 0 -F187 0.7501398636830259 0 -F188 -1.3840507191620501 0 -F189 -1.3863784044176497 0 -F190 0.35465073693029203 0 -F191 -0.0527678764658988 0 -F192 1.04452771030868 0 -F193 -1.0105215476872 0 -F194 -0.6098963926022061 0 -F195 1.24019945405587 0 -F196 -1.2486190178628 0 -F197 0.28873133501175297 0 -F198 -0.9444328759187892 0 -F199 -0.308547237330663 0 -======= ->>>>>>> 623c9aad86e777c1029ff286697e7d532a101258 diff --git a/q2_types/feature_data/tests/data/inf_differential.tsv b/q2_types/feature_data/tests/data/inf_differential.tsv index 5f9c8471..a2e56deb 100644 --- a/q2_types/feature_data/tests/data/inf_differential.tsv +++ b/q2_types/feature_data/tests/data/inf_differential.tsv @@ -26,7 +26,7 @@ F23 -1.11912925367142 F24 -1.10593563527746 F25 0.678934047810573 F26 -0.937189288219405 -pF27 -0.99973019311648 +F27 -0.99973019311648 F28 -0.31799033232181 F29 -0.66141741897569 F30 0.550511528315366 diff --git a/q2_types/feature_data/tests/test_transformer.py b/q2_types/feature_data/tests/test_transformer.py index 6b8714c2..2e730242 100644 --- a/q2_types/feature_data/tests/test_transformer.py +++ b/q2_types/feature_data/tests/test_transformer.py @@ -928,6 +928,43 @@ def test_aligned_rna_fasta_format_to_rna_iterator(self): self.assertEqual(observed, expected) +class TestDifferentialTransformer(TestPluginBase): + package = 'q2_types.feature_data.tests' + + def test_differential_to_df(self): + + _, obs = self.transform_format(DifferentialFormat, pd.DataFrame, + filename='differentials.tsv') + + # sniff to see if the first 4 feature ids are the same + exp = ['F0', 'F1', 'F2', 'F3'] + obs = list(obs.index[:4]) + self.assertListEqual(exp, obs) + + def test_differential_to_md(self): + + _, obs = self.transform_format(DifferentialFormat, qiime2.Metadata, + filename='differentials.tsv') + obs = obs.to_dataframe() + # sniff to see if the first 4 feature ids are the same + exp = ['F0', 'F1', 'F2', 'F3'] + obs = list(obs.index[:4]) + self.assertListEqual(exp, obs) + + def test_df_to_differential(self): + transformer = self.get_transformer(pd.DataFrame, DifferentialFormat) + + index = pd.Index(['SEQUENCE1', 'SEQUENCE2', 'SEQUENCE3']) + index.name = 'featureid' + input = pd.DataFrame( + [-1.3, 0.1, 1.2], index=index, columns=['differential'], + dtype=float) + + obs = transformer(input) + + self.assertIsInstance(obs, DifferentialFormat) + + class TestProteinFASTAFormatTransformers(TestPluginBase): package = 'q2_types.feature_data.tests' @@ -1127,29 +1164,5 @@ def test_aligned_protein_fasta_format_to_protein_iterator(self): self.assertEqual(observed, expected) -class TestDifferentialTransformer(TestPluginBase): - package = 'q2_types.feature_data.tests' - - def test_differential_to_df(self): - - _, obs = self.transform_format(DifferentialFormat, pd.DataFrame, - filename='differentials.tsv') - - # sniff to see if the first 4 feature ids are the same - exp = ['F0', 'F1', 'F2', 'F3'] - obs = list(obs.index[:4]) - self.assertListEqual(exp, obs) - - def test_df_to_differential(self): - transformer = self.get_transformer(pd.DataFrame, DifferentialFormat) - - index = pd.Index(['SEQUENCE1', 'SEQUENCE2', 'SEQUENCE3']) - input = pd.Series([-1.3, 0.1, 1.2], index=index, dtype=float) - - obs = transformer(input) - - self.assertIsInstance(obs, DifferentialFormat) - - if __name__ == '__main__': unittest.main() From 8c25f7458a2f1e66e4221b8f1dd2716b312b43bc Mon Sep 17 00:00:00 2001 From: mortonjt Date: Mon, 24 May 2021 12:15:52 -0600 Subject: [PATCH 14/15] adding in unittests --- .../tests/data/monte-carlo-samples.az | Bin 0 -> 15262 bytes q2_types/feature_data/tests/test_format.py | 2 ++ .../feature_data/tests/test_transformer.py | 26 ++++++++++++++++-- q2_types/feature_data/tests/test_type.py | 15 ++++++++-- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/q2_types/feature_data/tests/data/monte-carlo-samples.az b/q2_types/feature_data/tests/data/monte-carlo-samples.az index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..a7d87a59d0c0dc685312ad0cadc51a0249ef6029 100644 GIT binary patch literal 15262 zcmeHN2~bmK9)C#yIl>`=u#_XqR9!2Cg!_=o5CO{}r*>r%LI{`&1VbPkmQZ_1sYQwC zfW@P<*x|GaQlQ|hj#`g_q6Mr{cL7CFD7I(4}f?BV5WudY>h@Cwn&xvy)G5OM6@h{ zAVq@^M|9F988VSnERl{9OQx0x*hMYQw?WR0jDV=QCaw^LcL?$(Irw+|E`|MCJp8|s z;gj)@F6Q@S__*r}RNO@n;fd}E1~fzu(FU;#r6Pe$l*kvzkY(t)1^NRDjx0yEJgiF5f`B58(Ll7^YkB22KDY&O#o0aFAY zSNSO)V7cfc6p9I0r&7=c252LD8XXLc(8}V)^+PlbzzS^*m=oC)=q(3pR4OPH1(O6l z1uF-hxFHQQi$vlbLe@%S(OS{h08ttmG6oGN#6KzuO9|8vI1UI7<%UHChllZ_IQ}7A zgwDXR(8LwO{6o2OAP{a9h|>_n5LKPu_7aUupWZk0j83e*hya^inmr9{Fa+5fky zbR3W1e_AD*g{uUvNq}OaR3Jxi{C1k-Y335y9%FUY19Y{p^_O=7O{}tQ#3_sG4lc)) zdgf_Um5E_+Qr4I(ETl5Fc=>Mojz@{&YRw@+>5>iA=2Hw~#7Shcy z=mdV~TvX)c^~n$8Y?MmUMN*kql!2X`w&*rUo497Z9qrPvv%=e%>CR-?ZB0ya@n$pG zET*gK(7Dj)NG=y#SMzT&dcey9?I4jIxkW&PWlFBW9wGn{fCxYYAOaA9*9(Dco-n=u zO-UP}EcWUkT_odP`?(wBt0w+uz}!7Jg9tzbAOa8phyX<3^+4dCq|=)qA1dGmiNwe^ zf%$Wsc=;nidUX=guhND1(zN1V(S_QWK8;EXV&dY!4R8rwJV<_y_8SI-WJE~9#RQ=v zD5*(uY#fP1nkY^~$<>UxOdFaO!QH*ex)E#<2w=y^ggZt$VR2D3NNEi~R0#-HQw!ry zDH_BSp=9@(0%Zh|KCQ6?>NFEzb1B1@;9mR?XFbGOjVjOsa6(KEjTc0`UYwE=D@e%{ zEhxPZM~i!!R?y=BmqbGuP9tFHB|jPx92JfH0Tlz(DaZrkZWl-$VBKql9?%97eRU3U z3r_YLn?s*%BdByBh|y;$fb9Yyus{fr-1Xc@ZWxEl$3k#LZGsRI1u{W~NcJ+8L~*Jp z4dbk}0oAO5${+}1K|-1Y785}adsbGT!I-w+%`?aqH1$k#EIxuqhyX+YBJesP5XcD+ z!R~a}CsX`U0P+KOdCy0TrInvXZ0IQ4DQNqM?VnzcPM~9`z2aS;efoLf?xOIZkPS)Znw9aps7QL=vQ3Nqhk`XpB`lJ6f&1JIlAi@14L~Cq z2z&;Q5CMn)L;xZH5r7Cl1Rw(c?+9e`GHza~DR*>xC!#(>>*k8iOvB!=*3CEC% zy5X-C{MnS``u z-F2q>{#&Zvgn>tT36JuNHq%P4I+W@)ls(KDyZUMNVP?m!%$0E~21Awop?1|9l)Lxc zUHvE4!?-7RzPoD?`t4cZ#isg9e*fChGi5P>(MgT%kz;7QUx}Yi@jiLyBVX(O z7iE7Nkp3DIHyrxnP_rIyb|tSM z-RBTecHrKn3Gva32BY?M2S4Div8#|(2^By5T5`}j*XpUpH1)mw9b0E=TWtlo>cxTxo!yWC*vc$ zfx6zN{Z4m_C+N3E|8ml6w@Llh(2C?Th1<+|rLS>L)KF6D&*}G`xlTrx>8zoS)~b$- zXV+wWIoK%nvm=*)o`o4g|}EdsT3g_ks}EGF&T$7?Ud**s4uX%*}}*`2E{x0*>_ zSz7Pgc0?q6-a4o(KK6!#m+bMoJ6k8tUiIGo?zQDJ=hA{l#(%VPdi++6s;#rLHeS^* zYSZVPqfmEMX|A%q^JLZyR7>Q%6PC|TsBd;SI9aFWJ-L6j!Y$7y z-y^+Xg*rdasPU7MC4N)`#_$Uo%YTp2ho|J%4F})XF*36i8FnMFhQ><@ O3nq6?BaEh_;(q{^@}|H5 literal 0 HcmV?d00001 diff --git a/q2_types/feature_data/tests/test_format.py b/q2_types/feature_data/tests/test_format.py index fbe35b92..c3dac904 100644 --- a/q2_types/feature_data/tests/test_format.py +++ b/q2_types/feature_data/tests/test_format.py @@ -40,6 +40,7 @@ def test_taxonomy_format_validate_positive(self): for filepath in filepaths: format = TaxonomyFormat(filepath, mode='r') + format.validate() def test_taxonomy_format_validate_negative(self): @@ -137,6 +138,7 @@ def test_tsv_taxonomy_directory_format(self): os.path.join(self.temp_dir.name, 'taxonomy.tsv')) format = TSVTaxonomyDirectoryFormat(self.temp_dir.name, mode='r') + format.validate() def test_tsv_taxonomy_format_column_header_lengths(self): diff --git a/q2_types/feature_data/tests/test_transformer.py b/q2_types/feature_data/tests/test_transformer.py index 2e730242..89c75482 100644 --- a/q2_types/feature_data/tests/test_transformer.py +++ b/q2_types/feature_data/tests/test_transformer.py @@ -14,14 +14,16 @@ import biom import skbio import qiime2 - +import arviz as az +import numpy as np from pandas.testing import assert_frame_equal, assert_series_equal from q2_types.feature_table import BIOMV210Format from q2_types.feature_data import ( TaxonomyFormat, HeaderlessTSVTaxonomyFormat, TSVTaxonomyFormat, DNAFASTAFormat, DNAIterator, PairedDNAIterator, PairedDNASequencesDirectoryFormat, AlignedDNAFASTAFormat, - DifferentialFormat, AlignedDNAIterator, ProteinFASTAFormat, + DifferentialFormat, MonteCarloTensorFormat, + AlignedDNAIterator, ProteinFASTAFormat, AlignedProteinFASTAFormat, RNAFASTAFormat, AlignedRNAFASTAFormat, RNAIterator, AlignedRNAIterator ) @@ -965,6 +967,26 @@ def test_df_to_differential(self): self.assertIsInstance(obs, DifferentialFormat) +class TestMonteCarloTensorTransformer(TestPluginBase): + package = 'q2_types.feature_data.tests' + + def test_monte_carlo_tensor_to_infdata(self): + # check if object can be loaded + input, obs = self.transform_format( + MonteCarloTensorFormat, az.InferenceData, + filename='monte-carlo-samples.az') + exp = az.InferenceData.from_netcdf(str(input)) + self.assertEqual(exp.to_dataframe().shape, + obs.to_dataframe().shape) + + def test_infdata_to_monte_carlo_tensor(self): + transformer = self.get_transformer( + az.InferenceData, MonteCarloTensorFormat) + input = az.convert_to_inference_data(np.random.randn(100)) + obs = transformer(input) + self.assertIsInstance(obs, MonteCarloTensorFormat) + + class TestProteinFASTAFormatTransformers(TestPluginBase): package = 'q2_types.feature_data.tests' diff --git a/q2_types/feature_data/tests/test_type.py b/q2_types/feature_data/tests/test_type.py index 1f8457ef..05c62f19 100644 --- a/q2_types/feature_data/tests/test_type.py +++ b/q2_types/feature_data/tests/test_type.py @@ -10,8 +10,10 @@ from q2_types.feature_data import ( FeatureData, Taxonomy, Sequence, PairedEndSequence, AlignedSequence, - Differential, TSVTaxonomyDirectoryFormat, DNASequencesDirectoryFormat, - DifferentialDirectoryFormat, PairedDNASequencesDirectoryFormat, + Differential, MonteCarloTensor, + TSVTaxonomyDirectoryFormat, DNASequencesDirectoryFormat, + DifferentialDirectoryFormat, MonteCarloTensorDirectoryFormat, + PairedDNASequencesDirectoryFormat, AlignedDNASequencesDirectoryFormat, ProteinSequencesDirectoryFormat, AlignedProteinSequencesDirectoryFormat, ProteinSequence, AlignedProteinSequence, RNASequence, RNASequencesDirectoryFormat, @@ -40,7 +42,10 @@ def test_aligned_sequence_semantic_type_registration(self): self.assertRegisteredSemanticType(AlignedSequence) def test_differential_semantic_type_registration(self): - self.assertRegisteredSemanticType(AlignedSequence) + self.assertRegisteredSemanticType(Differential) + + def test_monte_carlo_tensor_semantic_type_registration(self): + self.assertRegisteredSemanticType(MonteCarloTensor) def test_protein_sequence_semantic_type_registration(self): self.assertRegisteredSemanticType(ProteinSequence) @@ -52,6 +57,10 @@ def test_differential_semantic_type_to_format_registration(self): self.assertSemanticTypeRegisteredToFormat( FeatureData[Differential], DifferentialDirectoryFormat) + def test_monte_carlo_tensor_semantic_type_to_format_registration(self): + self.assertSemanticTypeRegisteredToFormat( + MonteCarloTensor, MonteCarloTensorDirectoryFormat) + def test_taxonomy_semantic_type_to_format_registration(self): self.assertSemanticTypeRegisteredToFormat( FeatureData[Taxonomy], TSVTaxonomyDirectoryFormat) From f3783865b4fc03e5f4e930a5232f6969dc4867ee Mon Sep 17 00:00:00 2001 From: mortonjt Date: Mon, 24 May 2021 12:24:55 -0600 Subject: [PATCH 15/15] adding arviz dep --- ci/recipe/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/recipe/meta.yaml b/ci/recipe/meta.yaml index b114c3f7..81fcb989 100644 --- a/ci/recipe/meta.yaml +++ b/ci/recipe/meta.yaml @@ -25,6 +25,7 @@ requirements: - biom-format >=2.1.5,<2.2.0 - ijson - h5py + - arviz - qiime2 {{ release }}.* test: