Initial commit

snowwm · Jul 13, 2019 · 4de8846 · 4de8846
commit 4de8846
Show file tree

Hide file tree

Showing 11 changed files with 568 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,6 @@
+# Python
+__pycache__/
+
+# program output
+output.html
+records.yaml
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019-present Pavel Andreyev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
@@ -0,0 +1 @@
+Coming soon.
diff --git a/build/config.yaml b/build/config.yaml
@@ -0,0 +1,53 @@
+scraper:
+  url: http://www.radioscanner.ru/base/index.php?fromCity=%D1%E0%EC%E0%F0%E0&page={page}
+
+generator:
+  page_footer: По данным портала radioscanner.ru
+  table_caption: Частоты с {start_freq} по {end_freq} МГц  # TODO: make bold
+  columns_on_page: 2
+
+  columns:
+    - name: freq_date
+      class: strong
+      width: 12%
+      primary_header: Частота
+      primary_attr: frequency
+      secondary_header: Дата набл.
+      secondary_attr: date
+
+    - name: mod_type
+      class: strong
+      width: 12%
+      primary_header: Мод-ция
+      primary_attr: modulation
+      secondary_header: Тип сигн.
+      secondary_attr: signal_type
+
+    - name: service
+      width: 15%
+      primary_header: Служба радиосв.
+      primary_attr: service_type
+
+    - name: affil
+      width: 20%
+      primary_header: Принад-ть
+      primary_attr: affiliation
+
+    - name: call-sign
+      width: 17%
+      primary_header: Позывной
+      primary_attr: call_sign
+
+    - name: desc
+      primary_header: Описание
+      primary_attr: description
+
+  slices:
+    - 20
+    - 22
+    - 31
+    - 25
+    - 30
+    - 28
+    - 27
+    - remainder
diff --git a/build/output.html.mako b/build/output.html.mako
@@ -0,0 +1,64 @@
+## -*- coding: utf-8 -*-
+
+<link href="style.css" rel="stylesheet" type="text/css">
+
+<body>
+% for page in data:
+    <div class="page-container">
+        <div class="flex-container">
+        % for slice in page:
+            <div class="flex-item">
+                ${gen_table(slice)}
+            </div>
+        % endfor
+        </div>
+
+        <p class="footer">
+            ${page.footer}
+        </footer>
+    </div>
+% endfor
+</body>
+
+<%def name="gen_table(slice)">
+    <table>
+        <caption>
+            ${slice.caption}
+        </caption>
+        <thead>
+            <tr>
+            % for col in columns:
+                <th class="${col.get('class', '')}" style="width: ${col.get('width', 'initial')};">
+                    <p class="primary">
+                        ${col['primary_header']}
+                    </p>
+                    % if 'secondary_header' in col:
+                        <p class="secondary">
+                            ${col['secondary_header']}
+                        </p>
+                    % endif
+                </th>
+            % endfor
+            </tr>
+        </thead>
+
+        <tbody>
+        % for row in slice:
+            <tr>
+            % for col in columns:
+                <td class="${col.get('class', '')}">
+                    <p class="primary">
+                        ${row[col['primary_attr']] | h}
+                    </p>
+                    % if 'secondary_attr' in col:
+                        <p class="secondary">
+                            ${row[col['secondary_attr']] | h}
+                        </p>
+                    % endif
+                </td>
+            % endfor
+            </tr>
+        % endfor
+        </tbody>
+    </table>
+</%def>
diff --git a/build/style.css b/build/style.css
@@ -0,0 +1,83 @@
+/* Currently, this is a total mess. */
+
+body {
+    margin: 0;
+}
+
+.page-container {
+    width: 100vw;
+    height: 100vh;
+    contain: paint;
+}
+
+.flex-container {
+    display: flex;
+    align-items: center;
+    justify-content: space-evenly;
+    width: 100%;
+    height: 100%;
+}
+
+.footer {
+    position: absolute;
+    right: 0;
+    bottom: 0;
+    margin: 3mm 10mm;
+    text-align: right;
+    font-size: 14pt;
+    font-style: italic;
+}
+
+.flex-item {
+    width: calc(50% - 15mm);
+    height: calc(100% - 20mm);
+}
+
+table {
+    width: 100%;
+    height: calc(100% - 15mm);
+    table-layout: fixed;
+    border-collapse: collapse;
+    caption-side: top;
+}
+
+caption {
+    height: 15mm;
+    font-size: 24pt;
+}
+
+th, td {
+    padding: 1mm;
+    border: 0.5mm solid;
+}
+
+thead tr, tr:nth-child(even) {
+    background-color: #E8E8E8;
+}
+
+table p {
+    margin: 0;
+    hyphens: auto; /* rather disappointing: https://caniuse.com/#search=hyphens */
+    overflow: hidden;
+    overflow-wrap: break-word;
+    /* text-align: justify; */
+    /* text-overflow: ellipsis; /* it has some bugs, too */
+}
+
+.primary {
+    font-size: 22pt;
+}
+
+.secondary {
+    font-size: 18pt;
+    font-weight: normal;
+}
+
+.strong {
+    text-align: center;
+}
+
+.strong .primary {
+    font-weight: bold;
+    margin-bottom: 3mm;
+}
diff --git a/freq_table/__main__.py b/freq_table/__main__.py
@@ -0,0 +1,134 @@
+import logging
+import os
+from argparse import ArgumentParser
+
+import yaml
+
+from . import utils
+
+# TODO: write README
+# TODO: switch to OO style
+# TODO: auto make pdf
+# TODO: add record merging
+# TODO: add filters
+# TODO: add more output customization
+# TODO: improve page layout (a tricky task)
+# TODO: inline css?
+
+DEFAULT_CONFIG = 'build/config.yaml'
+DEFAULT_TEMPLATE = 'build/output.html.mako'
+DEFAULT_OUTPUT = 'build/output.html'
+DEFAULT_RECORDS = 'build/records.yaml'
+
+logger = logging.getLogger(__name__)
+if os.getenv('LOGLEVEL'):
+    level = getattr(logging, os.getenv('LOGLEVEL').upper())
+    logging.basicConfig(level=level)
+
+
+def parse_args():
+    parser = ArgumentParser(
+        prog='freq_table',
+        description='Make printable tables from radioscanner.ru frequency db.')
+
+    parser.add_argument('-s', '--scrape', action='store_true')
+    parser.add_argument('-u', '--update', action='store_true')
+    parser.add_argument('-n', '--no-gen', dest='gen_html',
+                        action='store_false')
+    parser.add_argument('-d', '--dump', nargs='?', const=DEFAULT_RECORDS,
+                        metavar='TO_FILE')
+    parser.add_argument('-l', '--load', nargs='?', const=DEFAULT_RECORDS,
+                        action='append', metavar='FROM_FILE')
+    parser.add_argument('-o', '--output', default=DEFAULT_OUTPUT,
+                        metavar='OUT_FILE')
+    parser.add_argument('-c', '--config', default=DEFAULT_CONFIG,
+                        metavar='CONF_FILE')
+    parser.add_argument('-t', '--template', default=DEFAULT_TEMPLATE,
+                        metavar='TEMPLATE_FILE')
+
+    args = parser.parse_args()
+
+    if args.update:
+        args.scrape = True
+
+    if not args.load:
+        if os.path.isfile(DEFAULT_RECORDS):
+            args.load = (DEFAULT_RECORDS,)
+            logger.warning('Using default records file.')
+        else:
+            args.scrape = True
+            logger.warning('No records file found, will scrape the web.')
+
+    return parser, args
+
+
+def add_records(store, records, from_='the web (may take a while)'):
+    logger.warning('Adding records from %s...', from_)
+    cnt = 0
+    for r in records:
+        store[r['url']] = r
+        cnt += 1
+    logger.warning('Added %i records.', cnt)
+
+
+def main():
+    parser, args = parse_args()
+
+    # load config
+    with open(args.config) as fh:
+        logger.warning('Loading config from %s...', fh.name)
+        utils.config = yaml.safe_load(fh)
+
+    # these imports use loaded config
+    from . import scraper
+    from . import generator
+    records_by_id = {}
+
+    # scrape records from the web
+    if args.scrape and not args.update:
+        add_records(records_by_id, scraper.get_records())
+
+    # load records from files
+    if args.load:
+        for file in args.load:
+            with open(file) as fh:
+                records = yaml.safe_load(fh)
+                add_records(records_by_id, records, from_=fh.name)
+
+    # scrape overwriting loaded records
+    if args.update:
+        add_records(records_by_id, scraper.get_records())
+
+    if not args.dump and not args.gen_html:
+        return
+
+    if len(records_by_id) == 0:
+        parser.error('No records to process. Aborting.')
+
+    logger.warning('Sorting records...')
+    records = sorted(records_by_id.values(),
+                     key=lambda r: float(r['frequency']))
+    logger.warning('Sorted %i records.', len(records))
+
+    # dump records to file
+    if args.dump:
+        with open(args.dump, 'w') as fh:
+            logger.warning('Writing records to %s...', fh.name)
+            yaml.safe_dump(records, fh, allow_unicode=True)
+
+    # and finally...
+    if args.gen_html:
+        with open(args.template) as fh:
+            logger.warning('Reading template from %s...', fh.name)
+            tmpl = fh.read()
+
+        with open(args.output, 'w') as fh:
+            logger.warning('Generating html at %s...', fh.name)
+            html = generator.generate_html(tmpl, records)
+            fh.write(html)
+
+    logger.warning('Done.')
+
+
+if __name__ == '__main__':
+    main()