diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..f7e74dd --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +[flake8] +max-line-length = 160 +ignore = W504, F401 +exclude = powerapi/test_utils \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..bae87cd --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,116 @@ +name: Release workflow + +on: + push: + tags: + - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10 +jobs: + check_version: + name: Check sensor version + runs-on: ubuntu-latest + outputs: + version: ${{ steps.step1.outputs.version }} + steps: + - uses: actions/checkout@v2 + - name: Check tag and sensor version + id: step1 + run: | + export VERSION=$(echo $GITHUB_REF | sed -e 's/refs\/tags\/v//g') + test $VERSION == $(grep Version control | cut -d ' ' -f 2) + echo "::set-output name=version::$VERSION" + build_pypi: + name: Push package on pypi + runs-on: ubuntu-latest + env: + PYPI_PASS: ${{ secrets.PYPI_PASS }} + needs: check_release + steps: + - uses: actions/checkout@v2 + - name: Prepare environement + run: pip install -U pip twine + - name: Init .pypirc + run: | + echo -e "[pypi]" >> ~/.pypirc + echo -e "username = powerapi" >> ~/.pypirc + echo -e "password = $PYPI_PASS" >> ~/.pypirc + - name: Generate package + run: | + python3 -m venv venv + . venv/bin/activate + python3 -m pip install wheel + python3 setup.py sdist bdist_wheel + - name: Upload to pypi + run: twine upload dist/* + build_docker_image: + name: Build and push docker image + runs-on: ubuntu-latest + needs: [check_version, build] + steps: + - uses: actions/checkout@v2 + - name: Log in to Docker Hub + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build and push image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: true + file: Dockerfile + tags: powerapi/procfs-sensor:latest, powerapi/procfs-sensor:${{needs.check_version.outputs.version}} + build_deb: + name: Build debian package + needs: [check_version, build] + runs-on: ubuntu-latest + container: + image: powerapi/procfs-sensor-build:latest + env: + VERSION: ${{needs.check_version.outputs.version}} + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + with: + name: binary + path: ~/bin/ + - name: Create package source + run: | + mkdir -p procfs-sensor-$VERSION/DEBIAN + cp control procfs-sensor-$VERSION/DEBIAN/ + mkdir -p procfs-sensor-$VERSION/usr/bin/ + cp ~/bin/procfs-sensor procfs-sensor-$VERSION/usr/bin/procfs-sensor + chmod +x procfs-sensor-$VERSION/usr/bin/procfs-sensor + dpkg-deb --build procfs-sensor-$VERSION + - uses: actions/upload-artifact@v2 + with: + name: deb_package + path: procfs-sensor-${{needs.check_version.outputs.version}}.deb + publish_release: + name: Publish release on github + runs-on: ubuntu-latest + needs: [check_version, build_deb] + env: + VERSION: ${{needs.check_version.outputs.version}} + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + with: + name: binary + path: ~/assets/ + - uses: actions/download-artifact@v2 + with: + name: deb_package + path: ~/assets/ + - name: Create Changelog + env: + CHANGELOG_CONTENT: ${{ steps.changelog.outputs.clean_changelog }} + run: | + sudo apt install -y npm + sudo npm install -g conventional-changelog-cli + conventional-changelog -p angular -r 2 | grep -v "^# \[\]" | sed 's/^#//g' > ~/final-changelog.md + cat ~/final-changelog.md + cat ~/final-changelog.md >> CHANGELOG.md + - name: Create Release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release create v$VERSION -d -t v$VERSION -F ~/final-changelog.md ~/assets/* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2ee49ae --- /dev/null +++ b/.gitignore @@ -0,0 +1,154 @@ +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Sphinx documentation +docs/_build/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +### Emacs ### +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### IntelliJ IDEA ### +.idea/ \ No newline at end of file diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..101df65 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,576 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS, test_utils + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=R0903, R0902, C0200, + C0114, C0103, R1705, R0913, + W0102, C0112, R1720, R0915, + W0237, R1732, W1514, R1718, + C0206, W0236, W0223, W1201, + W0622, W0212, C0209,R0914, + print-statement, + parameter-unpacking, + unpacking-in-except, + old-raise-syntax, + backtick, + long-suffix, + old-ne-operator, + old-octal-literal, + import-star-module-level, + non-ascii-bytes-literal, + raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + apply-builtin, + basestring-builtin, + buffer-builtin, + cmp-builtin, + coerce-builtin, + execfile-builtin, + file-builtin, + long-builtin, + raw_input-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + no-absolute-import, + old-division, + dict-iter-method, + dict-view-method, + next-method-called, + metaclass-assignment, + indexing-exception, + raising-string, + reload-builtin, + oct-method, + hex-method, + nonzero-method, + cmp-method, + input-builtin, + round-builtin, + intern-builtin, + unichr-builtin, + map-builtin-not-iterating, + zip-builtin-not-iterating, + range-builtin-not-iterating, + filter-builtin-not-iterating, + using-cmp-argument, + eq-without-hash, + div-method, + idiv-method, + rdiv-method, + exception-message-attribute, + invalid-str-codec, + sys-max-int, + bad-python3-import, + deprecated-string-function, + deprecated-str-translate-call, + deprecated-itertools-function, + deprecated-types-field, + next-method-defined, + dict-items-not-iterating, + dict-keys-not-iterating, + dict-values-not-iterating, + deprecated-operator-function, + deprecated-urllib-function, + xreadlines-attribute, + deprecated-sys-function, + exception-escape, + comprehension-escape + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package.. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[STRING] + +# This flag controls whether the implicit-str-concat-in-sequence should +# generate a warning on implicit string concatenation in sequences defined over +# several lines. +check-str-concat-over-line-jumps=no + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=160 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma, + dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[LOGGING] + +# Format style used to check logging format string. `old` means using % +# formatting, while `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement. +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=BaseException, + Exception diff --git a/Dockerfile-cpython b/Dockerfile-cpython new file mode 100644 index 0000000..0de3587 --- /dev/null +++ b/Dockerfile-cpython @@ -0,0 +1,6 @@ +FROM powerapi/powerapi:1.0.0 +USER powerapi +COPY --chown=powerapi . /tmp/procfs-sensor +RUN pip install --user --no-cache-dir "/tmp/procfs-sensor" && rm -r /tmp/procfs-sensor + +ENTRYPOINT ["python3", "-m", "procfs-sensor"] diff --git a/README.md b/README.md index c44ccb2..80ae358 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,31 @@ -# Procfs-sensor +VirtualWatts is a software-defined power meter based on the PowerAPI toolkit. +VirtualWatts a configurable software that can estimate the power consumption of +process inside a VM in real-time. +VirtuallWatts is supposed to be used inside a VM. It need to receive two metrics : -Proc file system Sensor is a tool that monitor the CPU usage of process using the proc -file system. - -The sensor use the proc file system of the Linux kernel. It is only available on Linux. - -The sensor could be used in a virtual machine. +- The energy consumption of the VM, usually made with + [SmartWatts](https://github.com/powerapi-ng/smartwatts-formula) +- The cpu usage of the tracked processes and the global cpu usage, as provided + by [procfs-sensor](https://github.com/powerapi-ng/procfs-sensor). # About -Procfs sensor is an open-source project developed by the [Spirals research +VirtualWatts is an open-source project developed by the [Spirals research group](https://team.inria.fr/spirals) (University of Lille 1 and Inria). The documentation is not available yet. + +## Contributing + +If you would like to contribute code you can do so through GitHub by forking the +repository and sending a pull request. +You should start by reading the [contribution guide](https://github.com/powerapi-ng/virtualwatts/blob/main/contributing.md) + +When submitting code, please check that it is conform by using `pylint` and +`flake8` with the configurations files at the root of the project. + +## Acknowledgments + +VirtualWatts is written in [Python](https://www.python.org/) (under [PSF +license](https://docs.python.org/3/license.html)) and built on top of +[PowerAPI](https://github.com/powerapi-ng/powerapi) diff --git a/contributing.md b/contributing.md new file mode 100644 index 0000000..8898fb3 --- /dev/null +++ b/contributing.md @@ -0,0 +1,38 @@ +# Contributing to Procfs-Sensor + +Thank you for your interest in contributing to Procfs-Sensor! There are multiple ways to contribute, and we appreciate all contributions. + +# Ways to contribute + +Pull requests should be made on the pre-release branch. This way they will be +included in the next stable release (available on master). +Each pull request should pass the tests and be correctly linted (`pylint` and +`flake8`) to pass the CI. + +Each pull request have to be reviewed before beeing accepted. Some modifications can be +asked by the projet's managers. + +Here is different ways to contribute to Procfs-Sensor. + +## Bug report + +If you are working with Procfs-Sensor and you encounter a bug, please let us know. +You are welcome to submit an issue which describe the bug and it context with a +small test or a code snippet that reproduce the issue. + +## Bug Fixes + +If you are interested to contribute to Procfs-Sensor you can fixes reported bugs. +They are listed in the issues with the label _bug_ + +Your pull request should contain a test that reproduce the bug, if the test do +not already exist, and the bug fix. +Please submit one pull request for each bug fix. + +## Patches + +You can submit bigger contributions in the form of patches. +A patch should : + +- include unit tests +- be an isolated changes. If you have uncorelated changes you should submit multiple pull requests to make it easier for us to review diff --git a/procfs_sensor/__init__.py b/procfs_sensor/__init__.py new file mode 100644 index 0000000..a3fc865 --- /dev/null +++ b/procfs_sensor/__init__.py @@ -0,0 +1,21 @@ +# Copyright (C) 2021 INRIA +# Copyright (C) 2021 University of Lille +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +""" +Declaration of module version and import of provided class +""" + +__version__ = "0.1.0" diff --git a/main.py b/procfs_sensor/__main__.py similarity index 74% rename from main.py rename to procfs_sensor/__main__.py index 339591b..907f3ff 100644 --- a/main.py +++ b/procfs_sensor/__main__.py @@ -34,6 +34,7 @@ import socket import threading + def read_config(config_file): """ Read the config from the config_files specified in argument""" file_object = open(config_file, "r") @@ -42,19 +43,18 @@ def read_config(config_file): return json.loads(json_content) - def mesure_cpu_usage(): """Mesure the cpu usage of process using pidstat""" - timestamp=datetime.today() + timestamp = datetime.today() raw_stat = str(subprocess.check_output(["pidstat"])) stat = raw_stat.split('\\n') pid_cpu_usage = {} - for i in range(3,len(stat)) : + for i in range(3, len(stat)): pid_stat = stat[i].split(' ') pid_stat = list(filter(('').__ne__, pid_stat)) - if len(pid_stat) != 10 : + if len(pid_stat) != 10: continue pid_cpu_usage[pid_stat[2]] = pid_stat[7] @@ -62,23 +62,22 @@ def mesure_cpu_usage(): return timestamp, pid_cpu_usage -def send_tcp_report(sock ,report): +def send_tcp_report(sock, report): """ Send the json report using TCP""" - to_send = report.encode('utf-8') sock.sendall(to_send) -def send_report(sock,report): +def send_report(sock, report): """ Send the json report using the output method specified in the config""" - send_tcp_report(sock,report) + send_tcp_report(sock, report) -def sensor_mesure_send(frequency,sensor,target,sock): +def sensor_mesure_send(sampling_interval, sensor, target, sock): """ Produce the report from scratch and send it""" - probe = threading.Timer(frequency/1000,sensor_mesure_send,[frequency,sensor,target,sock]) + probe = threading.Timer(sampling_interval / 1000, sensor_mesure_send, [sampling_interval, sensor, target, sock]) probe.start() timestamp, pid_cpu_usage = mesure_cpu_usage() @@ -86,50 +85,47 @@ def sensor_mesure_send(frequency,sensor,target,sock): cgroup_cpu_usage = {} global_cpu_usage = 0 - for process in pid_cpu_usage.keys(): - global_cpu_usage += float(pid_cpu_usage[process].replace(",",".")) + for process, _ in pid_cpu_usage: + global_cpu_usage += float(pid_cpu_usage[process].replace(", ", ".")) - - - for cgroup in target : + for cgroup in target: cgroup_cpu_usage[cgroup] = 0 cgroup_pid_file = open('/sys/fs/cgroup/perf_event/' + cgroup + '/tasks', "r") - cgroup_pid_raw =cgroup_pid_file.read() + cgroup_pid_raw = cgroup_pid_file.read() cgroup_pid_file.close() pid_list = cgroup_pid_raw.split('\n') for process in pid_list: if process not in pid_cpu_usage.keys(): continue - cgroup_cpu_usage[cgroup] += float(pid_cpu_usage[process].replace(",",".")) + cgroup_cpu_usage[cgroup] += float(pid_cpu_usage[process].replace(", ", ".")) - tmstp = str(timestamp) - tmstp[tmstp.index(' ')] = 'T' # Convert to mongo timestamp format - report = {'timestamp':tmstp, 'sensor':str(sensor), 'target':target, 'usage':cgroup_cpu_usage, "global_cpu_usage":global_cpu_usage} + tmstp_tmp = str(timestamp) + tmstp = tmstp_tmp[:tmstp_tmp.index(' ')] + 'T' + tmstp_tmp[tmstp_tmp.index(' ') + 1:] # Convert to mongo timestamp format + report = {'timestamp': tmstp, 'sensor': str(sensor), 'target': target, 'usage': cgroup_cpu_usage, "global_cpu_usage": global_cpu_usage} report_json = json.dumps(report) - send_report(sock,report_json) + send_report(sock, report_json) def main(): """Initialize the sensor and run it core function at the frequency required by the config""" - - if len(sys.argv) == 0 : - print("Precise config file name : ") + if len(sys.argv) == 1: + print("Precise config file name: ") file_name = input() else: file_name = sys.argv[1] - if len(file_name) < 5 : - logging.error("Error : the config file must be a .json") + if len(file_name) < 5: + logging.error("Error: the config file must be a .json") if file_name[-5:] != '.json': - logging.error("Error : the config file must be a .json") + logging.error("Error: the config file must be a .json") config = read_config(file_name) - sensor=config['name'] - target=config['target'] - frequency = int(config['frequency']) + sensor = config['name'] + target = config['target'] + sampling_interval = int(config['sampling-interval']) output = config['output'] @@ -140,7 +136,7 @@ def main(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((host, port)) - sensor_mesure_send(frequency, sensor,target,sock) + sensor_mesure_send(sampling_interval, sensor, target, sock) main() diff --git a/procfs_sensor/__pycache__/__init__.cpython-39.pyc b/procfs_sensor/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..1123727 Binary files /dev/null and b/procfs_sensor/__pycache__/__init__.cpython-39.pyc differ diff --git a/procfs_sensor/__pycache__/__main__.cpython-39.pyc b/procfs_sensor/__pycache__/__main__.cpython-39.pyc new file mode 100644 index 0000000..2127ef2 Binary files /dev/null and b/procfs_sensor/__pycache__/__main__.cpython-39.pyc differ diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..42500ca --- /dev/null +++ b/setup.cfg @@ -0,0 +1,24 @@ +[metadata] +name = procfs_sensor +version = attr: procfs_sensor.__version__ +description = Procfs-Sensor is a sensor that monitor CPU usage of cgroup +long_description = file: README.md, LICENSE +keywords = energy, powerapi, containers,sensor +author = Lauric Desauw +license = MIT +classifiers = + Programming Language :: Python :: 3.7 + License :: OSI Approved :: GNU Affero General Public License v3 + +[options] +zip_safe = False +include_package_data = True +python_requires = >= 3.7 +packages = find: +test_suite = tests +setup_requires = + pytest-runner >=3.9.2 +tests_require = + pytest >=3.9.2 + pytest-asyncio >=0.14.0 + pytest-timeout >= 1.4.2 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..1c57712 --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +# Copyright (C) 2021 INRIA +# Copyright (C) 2021 University of Lille +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from setuptools import setup +setup()