From 52b84047059db8f6219d57ddbc868fd38985dd6d Mon Sep 17 00:00:00 2001 From: juga0 Date: Sat, 24 Nov 2018 11:54:27 +0000 Subject: [PATCH 01/10] new: Add gitchangelog configuration file --- .gitchangelog.rc | 288 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 288 insertions(+) create mode 100644 .gitchangelog.rc diff --git a/.gitchangelog.rc b/.gitchangelog.rc new file mode 100644 index 00000000..45281f77 --- /dev/null +++ b/.gitchangelog.rc @@ -0,0 +1,288 @@ +# -*- coding: utf-8; mode: python -*- +## +## Format +## +## ACTION: [AUDIENCE:] COMMIT_MSG [!TAG ...] +## +## Description +## +## ACTION is one of 'chg', 'fix', 'new' +## +## Is WHAT the change is about. +## +## 'chg' is for refactor, small improvement, cosmetic changes... +## 'fix' is for bug fixes +## 'new' is for new features, big improvement +## +## AUDIENCE is optional and one of 'dev', 'usr', 'pkg', 'test', 'doc' +## +## Is WHO is concerned by the change. +## +## 'dev' is for developpers (API changes, refactors...) +## 'usr' is for final users (UI changes) +## 'pkg' is for packagers (packaging changes) +## 'test' is for testers (test only related changes) +## 'doc' is for doc guys (doc only changes) +## +## COMMIT_MSG is ... well ... the commit message itself. +## +## TAGs are additionnal adjective as 'refactor' 'minor' 'cosmetic' +## +## They are preceded with a '!' or a '@' (prefer the former, as the +## latter is wrongly interpreted in github.) Commonly used tags are: +## +## 'refactor' is obviously for refactoring code only +## 'minor' is for a very meaningless change (a typo, adding a comment) +## 'cosmetic' is for cosmetic driven change (re-indentation, 80-col...) +## 'wip' is for partial functionality but complete subfunctionality. +## +## Example: +## +## new: usr: support of bazaar implemented +## chg: re-indentend some lines !cosmetic +## new: dev: updated code to be compatible with last version of killer lib. +## fix: pkg: updated year of licence coverage. +## new: test: added a bunch of test around user usability of feature X. +## fix: typo in spelling my name in comment. !minor +## +## Please note that multi-line commit message are supported, and only the +## first line will be considered as the "summary" of the commit message. So +## tags, and other rules only applies to the summary. The body of the commit +## message will be displayed in the changelog without reformatting. + + +## +## ``ignore_regexps`` is a line of regexps +## +## Any commit having its full commit message matching any regexp listed here +## will be ignored and won't be reported in the changelog. +## +ignore_regexps = [ + r'@minor', r'!minor', + r'@cosmetic', r'!cosmetic', + r'@refactor', r'!refactor', + r'@wip', r'!wip', + r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*[p|P]kg:', + r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*[d|D]ev:', + r'^(.{3,3}\s*:)?\s*[fF]irst commit.?\s*$', + r'^$', ## ignore commits with empty messages +] + + +## ``section_regexps`` is a list of 2-tuples associating a string label and a +## list of regexp +## +## Commit messages will be classified in sections thanks to this. Section +## titles are the label, and a commit is classified under this section if any +## of the regexps associated is matching. +## +## Please note that ``section_regexps`` will only classify commits and won't +## make any changes to the contents. So you'll probably want to go check +## ``subject_process`` (or ``body_process``) to do some changes to the subject, +## whenever you are tweaking this variable. +## +section_regexps = [ + ('New', [ + r'^[nN]ew\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$', + ]), + ('Changes', [ + r'^[cC]hg\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$', + ]), + ('Fix', [ + r'^[fF]ix\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$', + ]), + + ('Other', None ## Match all lines + ), + +] + + +## ``body_process`` is a callable +## +## This callable will be given the original body and result will +## be used in the changelog. +## +## Available constructs are: +## +## - any python callable that take one txt argument and return txt argument. +## +## - ReSub(pattern, replacement): will apply regexp substitution. +## +## - Indent(chars=" "): will indent the text with the prefix +## Please remember that template engines gets also to modify the text and +## will usually indent themselves the text if needed. +## +## - Wrap(regexp=r"\n\n"): re-wrap text in separate paragraph to fill 80-Columns +## +## - noop: do nothing +## +## - ucfirst: ensure the first letter is uppercase. +## (usually used in the ``subject_process`` pipeline) +## +## - final_dot: ensure text finishes with a dot +## (usually used in the ``subject_process`` pipeline) +## +## - strip: remove any spaces before or after the content of the string +## +## - SetIfEmpty(msg="No commit message."): will set the text to +## whatever given ``msg`` if the current text is empty. +## +## Additionally, you can `pipe` the provided filters, for instance: +#body_process = Wrap(regexp=r'\n(?=\w+\s*:)') | Indent(chars=" ") +#body_process = Wrap(regexp=r'\n(?=\w+\s*:)') +#body_process = noop +body_process = ReSub(r'((^|\n)[A-Z]\w+(-\w+)*: .*(\n\s+.*)*)+$', r'') | strip + + +## ``subject_process`` is a callable +## +## This callable will be given the original subject and result will +## be used in the changelog. +## +## Available constructs are those listed in ``body_process`` doc. +subject_process = (strip | + ReSub(r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n@]*)(@[a-z]+\s+)*$', r'\4') | + SetIfEmpty("No commit message.") | ucfirst | final_dot) + + +## ``tag_filter_regexp`` is a regexp +## +## Tags that will be used for the changelog must match this regexp. +## +## tag_filter_regexp = r'^[0-9]+\.[0-9]+(\.[0-9]+)?$' +tag_filter_regexp = r'^v[0-9]+\.[0.9]+(\.[0-9]+)?$' + +## ``unreleased_version_label`` is a string or a callable that outputs a string +## +## This label will be used as the changelog Title of the last set of changes +## between last valid tag and HEAD if any. +unreleased_version_label = "Unreleased" + +## ``output_engine`` is a callable +## +## This will change the output format of the generated changelog file +## +## Available choices are: +## +## - rest_py +## +## Legacy pure python engine, outputs ReSTructured text. +## This is the default. +## +## - mustache() +## +## Template name could be any of the available templates in +## ``templates/mustache/*.tpl``. +## Requires python package ``pystache``. +## Examples: +## - mustache("markdown") +## - mustache("restructuredtext") +## +## - makotemplate() +## +## Template name could be any of the available templates in +## ``templates/mako/*.tpl``. +## Requires python package ``mako``. +## Examples: +## - makotemplate("restructuredtext") +## +output_engine = rest_py +#output_engine = mustache("restructuredtext") +#output_engine = mustache("markdown") +#output_engine = makotemplate("restructuredtext") + + +## ``include_merge`` is a boolean +## +## This option tells git-log whether to include merge commits in the log. +## The default is to include them. +include_merge = True + + +## ``log_encoding`` is a string identifier +## +## This option tells gitchangelog what encoding is outputed by ``git log``. +## The default is to be clever about it: it checks ``git config`` for +## ``i18n.logOutputEncoding``, and if not found will default to git's own +## default: ``utf-8``. +#log_encoding = 'utf-8' + + +## ``publish`` is a callable +## +## Sets what ``gitchangelog`` should do with the output generated by +## the output engine. ``publish`` is a callable taking one argument +## that is an interator on lines from the output engine. +## +## Some helper callable are provided: +## +## Available choices are: +## +## - stdout +## +## Outputs directly to standard output +## (This is the default) +## +## - FileInsertAtFirstRegexMatch(file, pattern, idx=lamda m: m.start(), flags) +## +## Creates a callable that will parse given file for the given +## regex pattern and will insert the output in the file. +## ``idx`` is a callable that receive the matching object and +## must return a integer index point where to insert the +## the output in the file. Default is to return the position of +## the start of the matched string. +## +## - FileRegexSubst(file, pattern, replace, flags) +## +## Apply a replace inplace in the given file. Your regex pattern must +## take care of everything and might be more complex. Check the README +## for a complete copy-pastable example. +## +publish = FileInsertAtFirstRegexMatch( + "CHANGELOG.rst", + r"(?Pv[0-9]+\.[0-9]+(\.[0-9]+)?)\s+\([0-9]+-[0-9]{2}-[0-9]{2}\)\n--+\n", + idx=lambda m: m.start(1) +) +#publish = stdout + + +## ``revs`` is a list of callable or a list of string +## +## callable will be called to resolve as strings and allow dynamical +## computation of these. The result will be used as revisions for +## gitchangelog (as if directly stated on the command line). This allows +## to filter exaclty which commits will be read by gitchangelog. +## +## To get a full documentation on the format of these strings, please +## refer to the ``git rev-list`` arguments. There are many examples. +## +## Using callables is especially useful, for instance, if you +## are using gitchangelog to generate incrementally your changelog. +## +## Some helpers are provided, you can use them:: +## +## - FileFirstRegexMatch(file, pattern): will return a callable that will +## return the first string match for the given pattern in the given file. +## If you use named sub-patterns in your regex pattern, it'll output only +## the string matching the regex pattern named "rev". +## +## - Caret(rev): will return the rev prefixed by a "^", which is a +## way to remove the given revision and all its ancestor. +## +## Please note that if you provide a rev-list on the command line, it'll +## replace this value (which will then be ignored). +## +## If empty, then ``gitchangelog`` will act as it had to generate a full +## changelog. +## +## The default is to use all commits to make the changelog. +#revs = ["^1.0.3", ] +revs = [ + Caret( + FileFirstRegexMatch( + "CHANGELOG.rst", + r"(?Pv[0-9]+\.[0-9]+(\.[0-9]+)?)\s+\([0-9]+-[0-9]{2}-[0-9]{2}\)\n--+\n")), + "HEAD" +] +#revs = [] From 269c99b2ee35123091acd8564ecf59db2d8fd61b Mon Sep 17 00:00:00 2001 From: juga0 Date: Sat, 24 Nov 2018 14:21:53 +0000 Subject: [PATCH 02/10] new: Add gitchangelog template --- .gitchangelogrst.tpl | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .gitchangelogrst.tpl diff --git a/.gitchangelogrst.tpl b/.gitchangelogrst.tpl new file mode 100644 index 00000000..46f02a39 --- /dev/null +++ b/.gitchangelogrst.tpl @@ -0,0 +1,34 @@ +% if data["title"]: +${data["title"]} +\n +${"=" * len(data["title"])} + + +% endif +% for version in data["versions"]: +<% +title = "%s (%s)" % (version["tag"], version["date"]) if version["tag"] else opts["unreleased_version_label"] + +nb_sections = len(version["sections"]) +%>${title} +${"-" * len(title)} +% for section in version["sections"]: +% if not (section["label"] == "Other" and nb_sections == 1): + +${section["label"]} +${"~" * len(section["label"])} +% endif +% for commit in section["commits"]: +<% +subject = "%s [%s]" % (commit["subject"], ", ".join(commit["authors"])) +entry = indent('\n'.join(textwrap.wrap(subject)), + first="- ").strip() +%>${entry} + +% if commit["body"]: +${indent(commit["body"])} +% endif +% endfor +% endfor + +% endfor From 2a7e6723ef6551e0dfd09f230846b6ff759e0cc0 Mon Sep 17 00:00:00 2001 From: juga0 Date: Sat, 24 Nov 2018 14:21:30 +0000 Subject: [PATCH 03/10] new: Convert changelog to rst Leave CHANGELOG.md until there's an actual new release, in case the unreleased changes are lost. Once CHANGELOG.md is removed, update the symlinks. --- CHANGELOG.rst | 331 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 331 insertions(+) create mode 100644 CHANGELOG.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 00000000..c917ace5 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,331 @@ +Changelog +========= + +All notable changes to this project will be documented in this file. + +The format is based on `Keep a +Changelog `__ and this project +adheres to `Semantic Versioning `__. + + +v1.0.2 (2018-11-10) +-------------------- + +Fixed +~~~~~ + +- Update bandwidth file specification version in the ``generator`` + (#28366). +- Use 5 "=" characters as terminator in the bandwidth files (#28379) + +Changed +~~~~~~~ + +- Include the headers about eligible relays in all the bandwidth files, + not only in the ones that does not have enough eligible relays + (#28365). + +v1.0.1 (2018-11-01) +-------------------- + +Changed +~~~~~~~ + +- Change default directories when sbws is run from a system service + (#28268). + +v1.0.0 (2018-10-29) +-------------------- + +**Important changes**: + +- ``generate`` includes extra statistics header lines when the number + of eligible relays to include is less than the 60% of the network. It + does not include the relays' lines. +- Speed up ``scanner`` by disabling RTT measurements and waiting for + measurement threads before prioritizing again the list of relays to + measure. + +Fixed +~~~~~ + +- Update python minimal version in setup (#28043) +- Catch unhandled exception when we fail to resolve a domain name + (#28141) +- Bandwidth filtered is the maximum between the bandwidth measurements + and their mean, not the minimum (#28215) +- Stop measuring the same relay by two threads(#28061) + +Changed +~~~~~~~ + +- Move ``examples/`` to ``docs/`` (#28040) +- Number of results comparison and number of results away from each + other are incorrect (#28041) +- Stop removing results that are not away from some other X secs + (#28103) +- Use secs-away when provided instead of data\_period (#28105) +- Disable measuring RTTs (#28159) +- Rename bandwidth file keyvalues (#28197) + +Added +----- + +- Write bw file only when the percentage of measured relays is bigger + than 60% (#28062) +- When the percentage of measured relays is less than the 60%, do not + include the relays in the bandwidth file and instead include some + statistics in the header (#28076) +- When the percentage of measured relays is less than the 60% and it + was more before, warn about it (#28155) +- When the difference between the total consensus bandwidth and the + total in the bandwidth lines is larger than 50%, warn (#28216) +- Add documentation about how the bandwidth measurements are selected + and scaled before writing them to the Bandwidth File (#27692) + +v0.8.0 (2018-10-08) +-------------------- + +**Important changes**: + +- Implement Torflow scaling/aggregation to be able to substitute + Torflow with sbws without affecting the bandwidth files results. +- Change stem dependency to 1.7.0, which removes the need for + \`dependency\_links\`\` +- Update and cleanup documentation + +Added +~~~~~ + +- Add system physical requirements section to INSTALL (#26937) +- Warn when there is not enough disk space (#26937) +- Implement Torflow scaling (#27108) +- Create methods to easy graph generation and obtain statistics to + compare with current torflow results.(#27688) +- Implement rounding bw in bandwidth files to 2 insignificant + digits(#27337) +- Filter results in order to include relays in the bandwidth file + that:(#27338) +- have at least two measured bandwidths +- the measured bandwidths are within 24 hours of each other +- have at least two descriptor observed bandwidths +- the descriptor observed bandwidths are within 24 hours of each other + +Fixed +~~~~~ + +- Broken environment variable in default sbws config. To use envvar + $FOO, write $$FOO in the config. +- Stop using directory as argument in integration tests (#27342) +- Fix typo getting configuration option to allow logging to file + (#27960) +- Set int type to new arguments that otherwise would be string (#27918) +- Stop printing arguments default values, since they are printed by + default (#27916) +- Use dash instead of underscore in new cli argument names (#27917) + +Changed +~~~~~~~ + +- sbws install doc is confusing (#27341) +- Include system and Python dependencies in ``INSTALL``. +- Include dependencies for docs and tests in ``INSTALL``. +- Point to ``DEPLOY`` to run sbws. +- Remove obsolete sections in ``INSTALL`` +- Simplify ``DEPLOY``, reuse terms in the ``glossary``. +- Remove obsolete ``sbws init`` from ``DEPLOY``. +- Point to config documentation. +- Add, unify and reuse terms in ``glossary``. +- refactor v3bwfile (#27386): move scaling method inside class +- use custom ``install_command`` to test installation commands while + ``dependency_links`` is needed until #26914 is fixed. (#27704) +- documentation cleanup (#27773) +- split, merge, simplify, extend, reorganize sections and files +- generate scales as Torflow by default (#27976) +- Replace stem ``dependency_links`` by stem 1.7.0 (#27705). This also + eliminates the need for custom ``install_command`` in tox. + +v0.7.0 (2018-08-09) +------------------- + +**Important changes**: + +- ``cleanup/stale_days`` is renamed to + ``cleanup/data_files_compress_after_days`` +- ``cleanup/rotten_days`` is renamed to + ``cleanup/data_files_delete_after_days`` +- sbws now takes as an argument the path to a config file (which + contains ``sbws_home``) instead of ``sbws_home`` (which contains the + path to a config file) + +Added +~~~~~ + +- Log line on start up with sbws version, platform info, and library + versions (trac#26751) +- Manual pages (#26926) + +Fixed +~~~~~ + +- Stop deleting the latest.v3bw symlink. Instead, do an atomic rename. + (#26740) +- State file for storing the last time ``sbws scanner`` was started, + and able to be used for storing many other types of state in the + future. (GH#166) +- Log files weren't rotating. Now they are. (#26881) + +Changed +~~~~~~~ + +- Remove test data v3bw file and generate it from the same test. + (#26736) +- Stop using food terms for cleanup-related config options +- Cleanup command now cleans up old v3bw files too (#26701) +- Make sbws more compatible with system packages: (#26862) +- Allow a configuration file argument +- Remove directory argument +- Create minimal user configuration when running +- Do not require to run a command to initialize +- Initialize directories when running +- Do not require configuration file inside directories specified by the + configuration + +v0.6.0 (2018-07-11) +------------------ + +**Important changes**: + +- The way users configure logging has changed. No longer are most users + expected to be familiar with how to configure python's standard + logging library with a config file. Instead we've abstracted out the + setting of log level, format, and destinations to make these settings + more accessible to users. Expert users familiar with `the logging + config file + format `__ + can still make tweaks. + +Summary of changes: + +- Make logging configuration easier for the user. +- Add UML diagrams to documentation. They can be found in + docs/source/images/ and regenerated with ``make umlsvg`` in docs/. + +Added +~~~~~ + +- UML diagrams to documentation. In docs/ run ``make umlsvg`` to + rebuild them. Requires graphviz to be installed.(GHPR#226) +- Add metadata to setup.py, useful for source/binary distributions. +- Add possibility to log to system log. (#26683) +- Add option to cleanup v3bw files. (#26701) + +Fixed +~~~~~ + +- Measure relays that have both Exit and BadExit as non-exits, which is + how clients would use them. (GH#217) +- Could not init sbws because of a catch-22 related to logging + configuration. Overhaul how logging is configured. (GH#186 GHPR#224) +- Call write method of V3BWFile class from the object instance. + (#26671) +- Stop calculating median on empty list .(#26666) + +Changed +~~~~~~~ + +- Remove is\_controller\_ok. Instead catch possible controller + exceptions and log them + +Removed +~~~~~~~ + +- Two parsing/plotting scripts in scripts/tools/ that can now be found + at https://github.com/pastly/v3bw-tools + +v0.5.0 (2018-06-26) +------------------ + +**Important changes**: + +- Result format changed, causing a version bump to 4. Updating sbws to + 0.5.0 will cause it to ignore results with version less than 4. + +Summary of changes: + +- Keep previously-generated v3bw files +- Allow a relay to limit its weight based on + RelayBandwidthRate/MaxAdvertisedBandwidth +- 1 CPU usage optimization +- 1 memory usage optimization + +Added +~~~~~ + +- Use a relay's {,Relay}BandwidthRate/MaxAdvertisedBandwidth as an + upper bound on the measurements we make for it. (GH#155) +- Ability to only consider results for a given relay valid if they came + from when that relay is using its most recent known IP address. + Thanks Juga. (GH#154 GHPR#199) +- Maintenance script to help us find functions that are (probably) no + longer being called. +- Integration test(s) for RelayPrioritizer (GHPR#206) +- Git/GitHub usage guidelines to CONTRIBUTING document (GH#208 + GHPR#215) + +Fixed +~~~~~ + +- Make relay priority calculations take only ~5% of the time they used + to (3s vs 60s) by using sets instead of lists when selecting + non-Authority relays. (GH#204) +- Make relay list refreshing take much less time by not allowing worker + threads to dogpile on the CPU. Before they would all start requesting + descriptors from Tor at roughly the same time, causing us to overload + our CPU core and make the process take unnecessarily long. Now we let + one thread do the work so it can peg the CPU on its own and get the + refresh done ASAP. (GH#205) +- Catch a JSON decode exception on malformed results so sbws can + continue gracefully (GH#210 GHPR#212) + +Changed +~~~~~~~ + +- Change the path where the Bandwidth List files are generated: now + they are stored in ``v3bw`` directory, named ``YYmmdd_HHMMSS.v3bw``, + and previously generated ones are kept. A ``latest.v3bw`` symlink is + updated. (GH#179 GHPR#190) +- Code refactoring in the v3bw classes and generation area +- Replace v3bw-into-xy bash script with python script to handle a more + complex v3bw file format (GH#182) + +v0.4.1 (2018-06-14) +------------------ + +Changed +~~~~~~~ + +- If the relay to measure is an exit, put it in the exit position and + choose a non-exit to help. Previously the relay to measure would + always be the first hop. (GH#181) +- Try harder to find a relay to help measure the target relay with two + changes. Essentially: (1) Instead of only picking from relays that + are 1.25 - 2.00 times faster than it by consensus weight, try (in + order) to find a relay that is at least 2.00, 1.75, 1.50, 1.25, or + v1.00 times as fast. If that fails, instead of giving up, (2) pick the + fastest relay in the network instead of giving up. This compliments + the previous change about measuring target exits in the exit + position. + +Fixed +~~~~~ + +- Exception that causes sbws to fall back to one measurement thread. We + first tried fixing something in this area with ``88fae60bc`` but + neglected to remember that ``.join()`` wants only string arguments + and can't handle a ``None``. So fix that. +- Exception when failing to get a relay's ``ed25519_master_key`` from + Tor and trying to do ``.rstrip()`` on a None. +- ``earliest_bandwidth`` being the newest bw not the oldest (thanks + juga0) +- ``node_id`` was missing the character "$" at the beginning From 0f7fc57d61658144c260b9d6f2f06104fd8c1abc Mon Sep 17 00:00:00 2001 From: juga0 Date: Wed, 6 Feb 2019 18:52:55 +0000 Subject: [PATCH 04/10] new: scripts: Add script to help new releases. --- scripts/maint/release.py | 43 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100755 scripts/maint/release.py diff --git a/scripts/maint/release.py b/scripts/maint/release.py new file mode 100755 index 00000000..a37bad72 --- /dev/null +++ b/scripts/maint/release.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Script to help release new versions.""" +import sys + + +def main(args): + version = args[0] + print("# This script will guide you to create a new release, but it will " + "not do anything.") + print("# ================================================================") + print("# Install gitchangelog in a virtualenv:") + print("pip install gitchangelog") + print("# Create a tag so that gitchangelog will create the new section " + "with the correct new tag:") + print("git tag {}".format(version)) + print("# Update the changelog:") + print("gitchangelog") + print("# Edit the changelog!") + print("# Remove the tag:") + print("git tag -d {}".format(version)) + print("# Replace the new version in __init__.py:") + print("# Commit __init__.py and CHANGELOG.rst with Release message:") + print("git commit -a -m 'Release {}'".format(version)) + print("# Create the final tag and sign it:") + print("git tag -s v{}".format(version)) + print("# Push now so that the Github tarball will have the correct tag.") + print("git push") + print("# Obtain Github tarball:") + print("torsocks wget " + "https://github.com/torproject/sbws/archive/v{}.tar.gz") + print("# Create the tarball signature:") + print("gpg --default-key F305447AF806D46B --detach-sign v{}.tar.gz " + "--output v{}.tar.gz.asc") + print("# Upload the signature manually to Github.") + print("# Now bump to the next development version:") + print("# Replace the version in __init__.py with the next version -dev0") + print("# Commit the development version:") + print("git commit -a -m 'Bump to version X'") + print("# Done!") + + +if __name__ == "__main__": + main(sys.argv[1:]) From 5046e2f35e4701dff70585c5b7a1d598221e53ce Mon Sep 17 00:00:00 2001 From: juga0 Date: Wed, 6 Feb 2019 19:05:21 +0000 Subject: [PATCH 05/10] new: docs: Include script on how to release --- CONTRIBUTING.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 72a0e78c..60b5cac4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -136,6 +136,9 @@ Commits Each commit should reference the Tor Project Trac ticket (example: ``#12345``) and possibly the bugfix version. +From version 1.0.3, the commit title should start by ``new`` or ``fix`` or +``change``. The commit message should contain ``Closes: #bugnumber``. + Try to make each commit a logically separate changes.:: As a general rule, your messages should start with a single line that’s @@ -258,6 +261,7 @@ Create a ./CHANGELOG.rst file. Each entry should reference the Tor Project Trac ticket (example: ``#12345``) and possibly the bugfix version. Until version 1.0.2 we have followed `keep a changelog`_ format. +From version 1.0.3, run ``./scripts/maint/release.py`` to create new releases. .. _commit-msg: From e54f087aafc78c2236915585808a6ab0adfe9065 Mon Sep 17 00:00:00 2001 From: juga0 Date: Wed, 6 Mar 2019 17:17:05 +0000 Subject: [PATCH 06/10] fixup! new: Convert changelog to rst --- CHANGELOG.rst | 331 -------------------------------------------------- 1 file changed, 331 deletions(-) delete mode 100644 CHANGELOG.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst deleted file mode 100644 index c917ace5..00000000 --- a/CHANGELOG.rst +++ /dev/null @@ -1,331 +0,0 @@ -Changelog -========= - -All notable changes to this project will be documented in this file. - -The format is based on `Keep a -Changelog `__ and this project -adheres to `Semantic Versioning `__. - - -v1.0.2 (2018-11-10) --------------------- - -Fixed -~~~~~ - -- Update bandwidth file specification version in the ``generator`` - (#28366). -- Use 5 "=" characters as terminator in the bandwidth files (#28379) - -Changed -~~~~~~~ - -- Include the headers about eligible relays in all the bandwidth files, - not only in the ones that does not have enough eligible relays - (#28365). - -v1.0.1 (2018-11-01) --------------------- - -Changed -~~~~~~~ - -- Change default directories when sbws is run from a system service - (#28268). - -v1.0.0 (2018-10-29) --------------------- - -**Important changes**: - -- ``generate`` includes extra statistics header lines when the number - of eligible relays to include is less than the 60% of the network. It - does not include the relays' lines. -- Speed up ``scanner`` by disabling RTT measurements and waiting for - measurement threads before prioritizing again the list of relays to - measure. - -Fixed -~~~~~ - -- Update python minimal version in setup (#28043) -- Catch unhandled exception when we fail to resolve a domain name - (#28141) -- Bandwidth filtered is the maximum between the bandwidth measurements - and their mean, not the minimum (#28215) -- Stop measuring the same relay by two threads(#28061) - -Changed -~~~~~~~ - -- Move ``examples/`` to ``docs/`` (#28040) -- Number of results comparison and number of results away from each - other are incorrect (#28041) -- Stop removing results that are not away from some other X secs - (#28103) -- Use secs-away when provided instead of data\_period (#28105) -- Disable measuring RTTs (#28159) -- Rename bandwidth file keyvalues (#28197) - -Added ------ - -- Write bw file only when the percentage of measured relays is bigger - than 60% (#28062) -- When the percentage of measured relays is less than the 60%, do not - include the relays in the bandwidth file and instead include some - statistics in the header (#28076) -- When the percentage of measured relays is less than the 60% and it - was more before, warn about it (#28155) -- When the difference between the total consensus bandwidth and the - total in the bandwidth lines is larger than 50%, warn (#28216) -- Add documentation about how the bandwidth measurements are selected - and scaled before writing them to the Bandwidth File (#27692) - -v0.8.0 (2018-10-08) --------------------- - -**Important changes**: - -- Implement Torflow scaling/aggregation to be able to substitute - Torflow with sbws without affecting the bandwidth files results. -- Change stem dependency to 1.7.0, which removes the need for - \`dependency\_links\`\` -- Update and cleanup documentation - -Added -~~~~~ - -- Add system physical requirements section to INSTALL (#26937) -- Warn when there is not enough disk space (#26937) -- Implement Torflow scaling (#27108) -- Create methods to easy graph generation and obtain statistics to - compare with current torflow results.(#27688) -- Implement rounding bw in bandwidth files to 2 insignificant - digits(#27337) -- Filter results in order to include relays in the bandwidth file - that:(#27338) -- have at least two measured bandwidths -- the measured bandwidths are within 24 hours of each other -- have at least two descriptor observed bandwidths -- the descriptor observed bandwidths are within 24 hours of each other - -Fixed -~~~~~ - -- Broken environment variable in default sbws config. To use envvar - $FOO, write $$FOO in the config. -- Stop using directory as argument in integration tests (#27342) -- Fix typo getting configuration option to allow logging to file - (#27960) -- Set int type to new arguments that otherwise would be string (#27918) -- Stop printing arguments default values, since they are printed by - default (#27916) -- Use dash instead of underscore in new cli argument names (#27917) - -Changed -~~~~~~~ - -- sbws install doc is confusing (#27341) -- Include system and Python dependencies in ``INSTALL``. -- Include dependencies for docs and tests in ``INSTALL``. -- Point to ``DEPLOY`` to run sbws. -- Remove obsolete sections in ``INSTALL`` -- Simplify ``DEPLOY``, reuse terms in the ``glossary``. -- Remove obsolete ``sbws init`` from ``DEPLOY``. -- Point to config documentation. -- Add, unify and reuse terms in ``glossary``. -- refactor v3bwfile (#27386): move scaling method inside class -- use custom ``install_command`` to test installation commands while - ``dependency_links`` is needed until #26914 is fixed. (#27704) -- documentation cleanup (#27773) -- split, merge, simplify, extend, reorganize sections and files -- generate scales as Torflow by default (#27976) -- Replace stem ``dependency_links`` by stem 1.7.0 (#27705). This also - eliminates the need for custom ``install_command`` in tox. - -v0.7.0 (2018-08-09) -------------------- - -**Important changes**: - -- ``cleanup/stale_days`` is renamed to - ``cleanup/data_files_compress_after_days`` -- ``cleanup/rotten_days`` is renamed to - ``cleanup/data_files_delete_after_days`` -- sbws now takes as an argument the path to a config file (which - contains ``sbws_home``) instead of ``sbws_home`` (which contains the - path to a config file) - -Added -~~~~~ - -- Log line on start up with sbws version, platform info, and library - versions (trac#26751) -- Manual pages (#26926) - -Fixed -~~~~~ - -- Stop deleting the latest.v3bw symlink. Instead, do an atomic rename. - (#26740) -- State file for storing the last time ``sbws scanner`` was started, - and able to be used for storing many other types of state in the - future. (GH#166) -- Log files weren't rotating. Now they are. (#26881) - -Changed -~~~~~~~ - -- Remove test data v3bw file and generate it from the same test. - (#26736) -- Stop using food terms for cleanup-related config options -- Cleanup command now cleans up old v3bw files too (#26701) -- Make sbws more compatible with system packages: (#26862) -- Allow a configuration file argument -- Remove directory argument -- Create minimal user configuration when running -- Do not require to run a command to initialize -- Initialize directories when running -- Do not require configuration file inside directories specified by the - configuration - -v0.6.0 (2018-07-11) ------------------- - -**Important changes**: - -- The way users configure logging has changed. No longer are most users - expected to be familiar with how to configure python's standard - logging library with a config file. Instead we've abstracted out the - setting of log level, format, and destinations to make these settings - more accessible to users. Expert users familiar with `the logging - config file - format `__ - can still make tweaks. - -Summary of changes: - -- Make logging configuration easier for the user. -- Add UML diagrams to documentation. They can be found in - docs/source/images/ and regenerated with ``make umlsvg`` in docs/. - -Added -~~~~~ - -- UML diagrams to documentation. In docs/ run ``make umlsvg`` to - rebuild them. Requires graphviz to be installed.(GHPR#226) -- Add metadata to setup.py, useful for source/binary distributions. -- Add possibility to log to system log. (#26683) -- Add option to cleanup v3bw files. (#26701) - -Fixed -~~~~~ - -- Measure relays that have both Exit and BadExit as non-exits, which is - how clients would use them. (GH#217) -- Could not init sbws because of a catch-22 related to logging - configuration. Overhaul how logging is configured. (GH#186 GHPR#224) -- Call write method of V3BWFile class from the object instance. - (#26671) -- Stop calculating median on empty list .(#26666) - -Changed -~~~~~~~ - -- Remove is\_controller\_ok. Instead catch possible controller - exceptions and log them - -Removed -~~~~~~~ - -- Two parsing/plotting scripts in scripts/tools/ that can now be found - at https://github.com/pastly/v3bw-tools - -v0.5.0 (2018-06-26) ------------------- - -**Important changes**: - -- Result format changed, causing a version bump to 4. Updating sbws to - 0.5.0 will cause it to ignore results with version less than 4. - -Summary of changes: - -- Keep previously-generated v3bw files -- Allow a relay to limit its weight based on - RelayBandwidthRate/MaxAdvertisedBandwidth -- 1 CPU usage optimization -- 1 memory usage optimization - -Added -~~~~~ - -- Use a relay's {,Relay}BandwidthRate/MaxAdvertisedBandwidth as an - upper bound on the measurements we make for it. (GH#155) -- Ability to only consider results for a given relay valid if they came - from when that relay is using its most recent known IP address. - Thanks Juga. (GH#154 GHPR#199) -- Maintenance script to help us find functions that are (probably) no - longer being called. -- Integration test(s) for RelayPrioritizer (GHPR#206) -- Git/GitHub usage guidelines to CONTRIBUTING document (GH#208 - GHPR#215) - -Fixed -~~~~~ - -- Make relay priority calculations take only ~5% of the time they used - to (3s vs 60s) by using sets instead of lists when selecting - non-Authority relays. (GH#204) -- Make relay list refreshing take much less time by not allowing worker - threads to dogpile on the CPU. Before they would all start requesting - descriptors from Tor at roughly the same time, causing us to overload - our CPU core and make the process take unnecessarily long. Now we let - one thread do the work so it can peg the CPU on its own and get the - refresh done ASAP. (GH#205) -- Catch a JSON decode exception on malformed results so sbws can - continue gracefully (GH#210 GHPR#212) - -Changed -~~~~~~~ - -- Change the path where the Bandwidth List files are generated: now - they are stored in ``v3bw`` directory, named ``YYmmdd_HHMMSS.v3bw``, - and previously generated ones are kept. A ``latest.v3bw`` symlink is - updated. (GH#179 GHPR#190) -- Code refactoring in the v3bw classes and generation area -- Replace v3bw-into-xy bash script with python script to handle a more - complex v3bw file format (GH#182) - -v0.4.1 (2018-06-14) ------------------- - -Changed -~~~~~~~ - -- If the relay to measure is an exit, put it in the exit position and - choose a non-exit to help. Previously the relay to measure would - always be the first hop. (GH#181) -- Try harder to find a relay to help measure the target relay with two - changes. Essentially: (1) Instead of only picking from relays that - are 1.25 - 2.00 times faster than it by consensus weight, try (in - order) to find a relay that is at least 2.00, 1.75, 1.50, 1.25, or - v1.00 times as fast. If that fails, instead of giving up, (2) pick the - fastest relay in the network instead of giving up. This compliments - the previous change about measuring target exits in the exit - position. - -Fixed -~~~~~ - -- Exception that causes sbws to fall back to one measurement thread. We - first tried fixing something in this area with ``88fae60bc`` but - neglected to remember that ``.join()`` wants only string arguments - and can't handle a ``None``. So fix that. -- Exception when failing to get a relay's ``ed25519_master_key`` from - Tor and trying to do ``.rstrip()`` on a None. -- ``earliest_bandwidth`` being the newest bw not the oldest (thanks - juga0) -- ``node_id`` was missing the character "$" at the beginning From d7a17c3e27dc63e090100b0222074d0d8bb5ba42 Mon Sep 17 00:00:00 2001 From: juga0 Date: Wed, 6 Mar 2019 17:04:14 +0000 Subject: [PATCH 07/10] fixup! new: Add gitchangelog configuration file --- .gitchangelog.rc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitchangelog.rc b/.gitchangelog.rc index 45281f77..3396c2c6 100644 --- a/.gitchangelog.rc +++ b/.gitchangelog.rc @@ -197,6 +197,8 @@ output_engine = rest_py ## ## This option tells git-log whether to include merge commits in the log. ## The default is to include them. +# Including merge commit to be able to detect which group of commits belong +# to the same feature/bug. include_merge = True From 3c329a2d724de91092e082f53908b31862f36bf8 Mon Sep 17 00:00:00 2001 From: juga0 Date: Wed, 6 Mar 2019 17:08:40 +0000 Subject: [PATCH 08/10] fixup! new: scripts: Add script to help new releases. --- scripts/maint/release.py | 197 ++++++++++++++++++++++++++++++++------- 1 file changed, 163 insertions(+), 34 deletions(-) diff --git a/scripts/maint/release.py b/scripts/maint/release.py index a37bad72..0b9d5e3e 100755 --- a/scripts/maint/release.py +++ b/scripts/maint/release.py @@ -1,42 +1,171 @@ #!/usr/bin/env python3 -"""Script to help release new versions.""" +# Copyright 2019 juga (juga at riseup dot net), CC0 license. +"""Script to help release new versions. + +It will: +0. Detect the current program version +1. Ask which version to release +2. Update the changelog automatically with ``gitchangelog`` + and leave the option to user to manually edit it +3. Modify the program version to release version, + commit it and commit changelog +4. Create a version with the tag and sign it +5. Push the commit and tag to the repository +6. Obtain the release tarball from Github +7. Sign the release tarball +8. Modify the program to the next prerelease version and commit it +9. Push the commit + +All in sequence and doesn't recover from any previous step. + +It assumes that: +- the program version is in ``__init__.py.__version__`` +- gitchangelog and semantic_version are installed +- we are in the master branch +- the remote repository to push is origin +- the next prerelease version is the release version + "-dev0" +- the official releases tarballs are in Github (because no access to dist.tpo) +- the key to sign the release is only one and is available in the system + +""" +import re +import subprocess import sys +try: + import semantic_version +except ImportError: + print("Please, install semantic_version") + sys.exit(1) +try: + import gitchangelog # noqa +except ImportError: + print("Please, install gitchangelog") + sys.exit(1) + +import sbws + + +def obtain_release_version(version): + # Ensure this is a prerelease version (eg 1.0.3-dev0) + assert version.prerelease + release_type_dict = { + 1: version.next_patch(), + 2: version.next_minor(), + 3: version.next_major(), + } + print("Current prerelease version: ", version) + print("which version would you like to release?:") + print("1. to patch release: ", release_type_dict[1]) + print("2. to minor release: ", release_type_dict[2]) + print("3. to major release: ", release_type_dict[3]) + release_type = int(input()) + try: + return release_type_dict[release_type] + except KeyError: + print("Invalid release.") + sys.exit(1) + + +def replace_version(old_version, new_version): + with open(sbws.__file__, 'r+') as f: + text = f.read() + text = re.sub(str(old_version), str(new_version), text) + f.seek(0) + f.write(text) + f.truncate() + + +def obtain_next_prerelease_version(release_version): + # Assuming that we are only jumping from release to `-dev0` + next_prerelease_version = semantic_version.Version( + str(release_version.next_patch()) + "-dev0" + ) + return next_prerelease_version def main(args): - version = args[0] - print("# This script will guide you to create a new release, but it will " - "not do anything.") - print("# ================================================================") - print("# Install gitchangelog in a virtualenv:") - print("pip install gitchangelog") - print("# Create a tag so that gitchangelog will create the new section " - "with the correct new tag:") - print("git tag {}".format(version)) - print("# Update the changelog:") - print("gitchangelog") - print("# Edit the changelog!") - print("# Remove the tag:") - print("git tag -d {}".format(version)) - print("# Replace the new version in __init__.py:") - print("# Commit __init__.py and CHANGELOG.rst with Release message:") - print("git commit -a -m 'Release {}'".format(version)) - print("# Create the final tag and sign it:") - print("git tag -s v{}".format(version)) - print("# Push now so that the Github tarball will have the correct tag.") - print("git push") - print("# Obtain Github tarball:") - print("torsocks wget " - "https://github.com/torproject/sbws/archive/v{}.tar.gz") - print("# Create the tarball signature:") - print("gpg --default-key F305447AF806D46B --detach-sign v{}.tar.gz " - "--output v{}.tar.gz.asc") - print("# Upload the signature manually to Github.") - print("# Now bump to the next development version:") - print("# Replace the version in __init__.py with the next version -dev0") - print("# Commit the development version:") - print("git commit -a -m 'Bump to version X'") - print("# Done!") + print(__doc__) + print("1. Which version to release") + print("---------------------------") + current_version = semantic_version.Version(sbws.__version__) + release_version = obtain_release_version(current_version) + + print("\n2. Update changelog") + print("-------------------") + print("Creating tag v{} so that gitchangelog will create the new section " + "with the correct new tag...".format(release_version)) + subprocess.call(['git', 'tag', 'v{}'.format(release_version), + '-m', '\"Release version {}.\"'.format(release_version)]) + + print("\nUpdating the changelog automatically...") + subprocess.call('gitchangelog'.split(' ')) + + print("\nEdit the changelog manually to remove merge commits and " + "leave only the apropiate paragraph for every bug/feature.") + input("Press enter when done.\n") + + print("\nRemoving the tag...") + subprocess.call(['git', 'tag', '-d', 'v{}'.format(release_version)]) + + print("\n3. Modify program version") + print("--------------------------") + print("\nReplacing __init__.py version with the release version...") + replace_version(current_version, release_version) + + print("\nCommiting __init__.py and CHANGELOG.rst...") + subprocess.call(['git', 'commit', + '-am', '"Release version {}."'.format(release_version)]) + + print("\n4. Create tag and sign it") + print("--------------------------") + print("\nCreating the final tag and signing it...") + subprocess.call(['git', 'tag', '-s', 'v{}'.format(release_version), + '-m', '"Release version {}."'.format(release_version)]) + + print("\n5. Push commit and tag") + print("----------------------") + print("\nPush now so that the Github tarball will have the" + " correct tag...") + input("Press enter when you are sure everything is correct.") + subprocess.call(['git', 'push', 'origin', 'master']) + + # While we use Github releases and not dist.tpo + print("\n6. Create release tarball") + print("-------------------------") + print("Obtaining Github tarball...") + subprocess.call( + "torsocks wget https://github.com/torproject/sbws/archive/v{}.tar.gz" + .format(release_version).split(' ') + ) + + print("\n7. Create the tarball signature") + print("-------------------------------") + print("Creating detached signature...") + subprocess.call("gpg --default-key F305447AF806D46B " + "--output v{}.tar.gz.asc " + "--detach-sign v{}.tar.gz" + .format(release_version, release_version).split(' ')) + + print("\nUpload the signature manually to Github.") + input("Press enter when done.") + print("\nRelease done!!!.") + + print("\n8. Create next prerelease version") + print("---------------------------------") + next_prerelease_version = obtain_next_prerelease_version(release_version) + print("\nReplacing the version in the program with " + "the next prerelease version...") + replace_version(release_version, next_prerelease_version) + + print("\nCommitting the prerelease version...") + subprocess.call(['git', 'commit', + '-am', '"Bump to {}."'.format(next_prerelease_version)]) + + print("\n9. Push commit") + print("--------------") + input("Press enter when you are sure everything is correct.") + subprocess.call(['git', 'push', 'origin', 'master']) + print("Done!") if __name__ == "__main__": From 6d3879ca4cd8cbc144c5bc22e8865bb8dfe85314 Mon Sep 17 00:00:00 2001 From: juga0 Date: Mon, 11 Mar 2019 08:45:01 +0000 Subject: [PATCH 09/10] fixup! fixup! new: scripts: Add script to help new releases. --- scripts/maint/release.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts/maint/release.py b/scripts/maint/release.py index 0b9d5e3e..e69933a1 100755 --- a/scripts/maint/release.py +++ b/scripts/maint/release.py @@ -132,12 +132,36 @@ def main(args): # While we use Github releases and not dist.tpo print("\n6. Create release tarball") print("-------------------------") + + print("\nCreating a release tarball...") + subprocess.call( + "git archive --format=tar.gz --prefix=sbws-{}/ " + "-o v{}.tar.gz v{}" + .format(release_version, release_version, release_version).split(' ') + ) + print("\nCreating tarball hash file...") + fd = open('v{}.tar.gz.sha256'.format(release_version), 'w') + subprocess.call("sha256sum v{}.tar.gz".format(release_version).split(' '), + stdout=fd) + fd.close() + print("Obtaining Github tarball...") + # This will overwrite local tarball, but that's fine since the hash file + # won't be overwritten. subprocess.call( - "torsocks wget https://github.com/torproject/sbws/archive/v{}.tar.gz" - .format(release_version).split(' ') + "wget https://github.com/torproject/sbws/archive/v{}.tar.gz " + "-O v{}.tar.gz" + .format(release_version, release_version).split(' ') ) + print("Verifying Github tarball and local one are the same...") + try: + subprocess.check_call("sha256sum -c v{}.tar.gz.sha256" + .format(release_version).split(' ')) + except subprocess.CalledProcessError: + print("Tarballs are not the same") + sys.exit(1) + print("\n7. Create the tarball signature") print("-------------------------------") print("Creating detached signature...") From c1d0263dafde2f9f80ea28741d1828b988794dc4 Mon Sep 17 00:00:00 2001 From: juga0 Date: Mon, 11 Mar 2019 09:22:33 +0000 Subject: [PATCH 10/10] fixup! fixup! new: scripts: Add script to help new releases. --- scripts/maint/release.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/maint/release.py b/scripts/maint/release.py index e69933a1..01e63e86 100755 --- a/scripts/maint/release.py +++ b/scripts/maint/release.py @@ -2,6 +2,8 @@ # Copyright 2019 juga (juga at riseup dot net), CC0 license. """Script to help release new versions. +usage: release.py GPG_KEY_IDENTIFIER + It will: 0. Detect the current program version 1. Ask which version to release @@ -85,6 +87,11 @@ def obtain_next_prerelease_version(release_version): def main(args): print(__doc__) + try: + keyid = args[0] + except IndexError: + print("Please, pass a GPG key identifier as argument.") + sys.exit(1) print("1. Which version to release") print("---------------------------") current_version = semantic_version.Version(sbws.__version__) @@ -165,10 +172,11 @@ def main(args): print("\n7. Create the tarball signature") print("-------------------------------") print("Creating detached signature...") - subprocess.call("gpg --default-key F305447AF806D46B " + subprocess.call("gpg --default-key {} " "--output v{}.tar.gz.asc " "--detach-sign v{}.tar.gz" - .format(release_version, release_version).split(' ')) + .format(keyid, release_version, release_version) + .split(' ')) print("\nUpload the signature manually to Github.") input("Press enter when done.")