2 # org.onap.vnfrqts/requirements
3 # ============LICENSE_START====================================================
4 # Copyright © 2018 AT&T Intellectual Property. All rights reserved.
6 # Unless otherwise specified, all software contained herein is licensed
7 # under the Apache License, Version 2.0 (the "License");
8 # you may not use this software except in compliance with the License.
9 # You may obtain a copy of the License at
11 # http://www.apache.org/licenses/LICENSE-2.0
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS,
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
19 # Unless otherwise specified, all documentation contained herein is licensed
20 # under the Creative Commons License, Attribution 4.0 Intl. (the "License");
21 # you may not use this documentation except in compliance with the License.
22 # You may obtain a copy of the License at
24 # https://creativecommons.org/licenses/by/4.0/
26 # Unless required by applicable law or agreed to in writing, documentation
27 # distributed under the License is distributed on an "AS IS" BASIS,
28 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
29 # See the License for the specific language governing permissions and
30 # limitations under the License.
32 # ============LICENSE_END============================================
35 This script will generate an summary of the requirements changes between
36 two version's of requirements by analyzing the needs.json file. The template
37 can be customized by updating release-requirement-changes.rst.jinja2.
40 from itertools import groupby, chain
48 from operator import itemgetter
52 NEEDS_JSON_URL = "https://nexus.onap.org/service/local/repositories/raw/content/org.onap.vnfrqts.requirements/master/needs.json"
53 NEEDS_PATH = "docs/data/needs.json"
54 JINJA_TEMPLATE = "release-requirement-changes.rst.jinja2"
57 def check(predicate, msg):
59 Raises a ``RuntimeError`` if the given predicate is false.
61 :param predicate: Predicate to evaluate
62 :param msg: Error message to use if predicate is false
65 raise RuntimeError(msg)
68 class DifferenceFinder:
70 Class takes a needs.json data structure and finds the differences
71 between two different versions of the requirements
74 def __init__(self, needs, current_version, prior_version):
76 Determine the differences between the ``current_version`` and the
77 ``prior_version`` of the given requirements.
79 :param needs: previously loaded needs.json file
80 :param current_version: most recent version to compare against
81 :param prior_version: a prior version
85 self.current_version = current_version
86 self.prior_version = prior_version
91 Validates the inputs to the ``DifferenceFinder`` constructor.
93 :raises RuntimeError: if the file is not structured properly or the
94 given versions can't be found.
96 check(self.needs is not None, "needs cannot be None")
97 check(isinstance(self.needs, dict), "needs must be be a dict")
98 check("versions" in self.needs, "needs file not properly formatted")
99 for version in (self.current_version, self.prior_version):
101 version in self.needs["versions"],
102 "Version " + version + " was not found in the needs file",
106 def current_requirements(self):
107 """Returns a dict of requirement ID to requirement metadata"""
108 return self.get_version(self.current_version)
111 def prior_requirements(self):
112 """Returns a dict of requirement ID to requirement metadata"""
113 return self.get_version(self.prior_version)
115 def get_version(self, version):
116 """Returns a dict of requirement ID to requirement metadata"""
117 return self.needs["versions"][version]["needs"]
120 def new_requirements(self):
121 """Requirements added since the prior version"""
122 new_ids = self.current_ids.difference(self.prior_ids)
123 return self.filter_needs(self.current_requirements, new_ids)
126 def current_ids(self):
127 """Returns a set of the requirement IDs for the current version"""
128 return set(self.current_requirements.keys())
132 """Returns a set of the requirement IDs for the prior version"""
133 return set(self.prior_requirements.keys())
136 def removed_requirements(self):
137 """Requirements that were removed since the prior version"""
138 removed_ids = self.prior_ids.difference(self.current_ids)
139 return self.filter_needs(self.prior_requirements, removed_ids)
142 def changed_requirements(self):
143 """"Requirements where the description changed since the last version"""
144 common_ids = self.prior_ids.intersection(self.current_ids)
146 for r_id in common_ids:
147 current_text = self.current_requirements[r_id]["description"]
148 prior_text = self.prior_requirements[r_id]["description"]
149 if not self.is_equivalent(current_text, prior_text):
150 sections = self.current_requirements[r_id]["sections"]
153 "description": current_text,
154 "sections": sections,
155 "introduced": self.current_requirements[r_id].get("introduced"),
156 "updated": self.current_requirements[r_id].get("updated")
160 def is_equivalent(self, current_text, prior_text):
161 """Returns true if there are meaningful differences between the
162 text. See normalize for more information"""
163 return self.normalize(current_text) == self.normalize(prior_text)
167 """Strips out formatting, line breaks, and repeated spaces to normalize
168 the string for comparison. This ensures minor formatting changes
169 are not tagged as meaningful changes"""
171 s = s.replace("\n", " ")
172 s = re.sub(r'[`*\'"]', "", s)
173 s = re.sub(r"\s+", " ", s)
177 def filter_needs(needs, ids):
179 Return the requirements with the given ids
181 :ids: sequence of requirement IDs
182 :returns: dict of requirement ID to requirement data for only the
183 requirements in ``ids``
185 return {r_id: data for r_id, data in needs.items() if r_id in ids}
188 def load_requirements(path):
189 """Load the requirements from the needs.json file"""
190 if not (os.path.exists(path)):
191 print("needs.json not found. Run tox -e docs to generate it.")
193 with open(path, "r") as req_file:
194 return json.load(req_file)
197 def load_current_requirements():
198 """Loads dict of current requirements or empty dict if file doesn't exist"""
200 r = requests.get(NEEDS_JSON_URL)
201 if r.headers.get("content-type") == "application/json":
202 with open(NEEDS_PATH, "wb") as needs:
203 needs.write(r.content)
207 "Unexpected content-type ({}) encountered downloading "
208 + "requirements.json, using last saved copy"
209 ).format(r.headers.get("content-type"))
211 except requests.exceptions.RequestException as e:
212 warnings.warn("Error downloading latest JSON, using last saved copy.")
213 warnings.warn(UserWarning(e))
214 with open(NEEDS_PATH, "r") as f:
218 """Parse the command-line arguments and return the arguments:
223 parser = argparse.ArgumentParser(
224 description="Generate RST summarizing requirement changes between "
225 "two given releases. The resulting RST file will be "
226 "written to the docs/ directory"
229 "current_version", help="Current release in lowercase (ex: casablanca)"
231 parser.add_argument("prior_version",
232 help="Prior release to compare against")
233 return parser.parse_args()
236 def tag(dicts, key, value):
237 """Adds the key value to each dict in the sequence"""
243 def gather_section_changes(diffs):
245 Return a list of dicts that represent the changes for a given section path.
248 "section_path": path,
250 "updated: [req, ...],
251 "removed: [req, ...],
255 :param diffs: instance of DifferenceFinder
256 :return: list of section changes
258 # Add "change" and "section_path" keys to all requirements
261 tag(diffs.new_requirements.values(), "change", "added"),
262 tag(diffs.changed_requirements.values(), "change", "updated"),
263 tag(diffs.removed_requirements.values(), "change", "removed"),
267 req["section_path"] = " > ".join(reversed(req["sections"]))
269 # Build list of changes by section
270 reqs = sorted(reqs, key=itemgetter("section_path"))
272 for section, section_reqs in groupby(reqs, key=itemgetter("section_path")):
273 change = itemgetter("change")
274 section_reqs = sorted(section_reqs, key=change)
275 section_changes = {"section_path": section}
276 for change, change_reqs in groupby(section_reqs, key=change):
277 section_changes[change] = list(change_reqs)
278 if any(k in section_changes for k in ("added", "updated", "removed")):
279 all_changes.append(section_changes)
283 def render_to_file(template_path, output_path, **context):
284 """Read the template and render it ot the output_path using the given
286 with open(template_path, "r") as infile:
287 t = jinja2.Template(infile.read())
288 result = t.render(**context)
289 with open(output_path, "w") as outfile:
290 outfile.write(result)
292 print("Writing requirement changes to " + output_path)
294 "Please be sure to update the docs/release-notes.rst document "
295 "with a link to this document"
299 def print_invalid_metadata_report(difference_finder, current_version):
300 """Write a report to the console for any instances where differences
301 are detected, but the appropriate :introduced: or :updated: metadata
302 is not applied to the requirement."""
303 print("Validating Metadata...")
305 print("Requirements Added, but Missing :introduced: Attribute")
306 print("----------------------------------------------------")
307 errors = [["reqt_id", "attribute", "value"]]
308 for req in difference_finder.new_requirements.values():
309 if "introduced" not in req or req["introduced"] != current_version:
310 errors.append([req["id"], ":introduced:", current_version])
313 print("Requirements Changed, but Missing :updated: Attribute")
314 print("-----------------------------------------------------")
315 for req in difference_finder.changed_requirements.values():
316 if "updated" not in req or req["updated"] != current_version:
317 errors.append([req["id"], ":updated:", current_version])
319 with open("invalid_metadata.csv", "w", newline="") as error_report:
320 error_report = csv.writer(error_report)
321 error_report.writerows(errors)
324 if __name__ == "__main__":
326 requirements = load_current_requirements()
327 differ = DifferenceFinder(requirements,
328 args.current_version,
331 print_invalid_metadata_report(differ, args.current_version)
333 changes = gather_section_changes(differ)
335 "release-requirement-changes.rst.jinja2",
336 "docs/changes-by-section-" + args.current_version + ".rst",
338 current_version=args.current_version,
339 prior_version=args.prior_version,
340 num_added=len(differ.new_requirements),
341 num_removed=len(differ.removed_requirements),
342 num_changed=len(differ.changed_requirements),