From 7b0d88ff0d497b0870e0430ec31bcc9becf7483e Mon Sep 17 00:00:00 2001 From: Ray Date: Fri, 6 Oct 2023 13:15:45 -0600 Subject: [PATCH 1/8] Add basic free-busy report --- radicale/app/report.py | 63 +++++++++++++++++++++++----- radicale/item/filter.py | 79 ++++++++++++++++++++++++------------ radicale/storage/__init__.py | 18 ++++++++ radicale/tests/__init__.py | 19 +++++++-- radicale/tests/test_base.py | 20 ++++++++- 5 files changed, 156 insertions(+), 43 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 8a9474321..1ec32b068 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -23,6 +23,7 @@ import posixpath import socket import xml.etree.ElementTree as ET +import vobject from http import client from typing import (Any, Callable, Iterable, Iterator, List, Optional, Sequence, Tuple, Union) @@ -37,12 +38,38 @@ from radicale.item import filter as radicale_filter from radicale.log import logger +def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], + collection: storage.BaseCollection, encoding: str, + unlock_storage_fn: Callable[[], None] + ) -> Tuple[int, str]: + multistatus = ET.Element(xmlutils.make_clark("D:multistatus")) + if xml_request is None: + return client.MULTI_STATUS, multistatus + root = xml_request + if (root.tag == xmlutils.make_clark("C:free-busy-query") and + collection.tag != "VCALENDAR"): + logger.warning("Invalid REPORT method %r on %r requested", + xmlutils.make_human_tag(root.tag), path) + return client.FORBIDDEN, xmlutils.webdav_error("D:supported-report") + + time_range_element = root.find(xmlutils.make_clark("C:time-range")) + start,end = radicale_filter.time_range_timestamps(time_range_element) + items = list(collection.get_by_time(start, end)) + + cal = vobject.iCalendar() + for item in items: + occurrences = radicale_filter.time_range_fill(item.vobject_item, time_range_element, "VEVENT") + for occurrence in occurrences: + vfb = cal.add('vfreebusy') + vfb.add('dtstamp').value = item.vobject_item.vevent.dtstamp.value + vfb.add('dtstart').value, vfb.add('dtend').value = occurrence + return (client.OK, cal.serialize()) def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], collection: storage.BaseCollection, encoding: str, unlock_storage_fn: Callable[[], None] ) -> Tuple[int, ET.Element]: - """Read and answer REPORT requests. + """Read and answer REPORT requests that return XML. Read rfc3253-3.6 for info. @@ -426,13 +453,27 @@ def do_REPORT(self, environ: types.WSGIEnviron, base_prefix: str, else: assert item.collection is not None collection = item.collection - try: - status, xml_answer = xml_report( - base_prefix, path, xml_content, collection, self._encoding, - lock_stack.close) - except ValueError as e: - logger.warning( - "Bad REPORT request on %r: %s", path, e, exc_info=True) - return httputils.BAD_REQUEST - headers = {"Content-Type": "text/xml; charset=%s" % self._encoding} - return status, headers, self._xml_response(xml_answer) + + if xml_content is not None and \ + xml_content.tag == xmlutils.make_clark("C:free-busy-query"): + try: + status, body = free_busy_report( + base_prefix, path, xml_content, collection, self._encoding, + lock_stack.close) + except ValueError as e: + logger.warning( + "Bad REPORT request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + headers = {"Content-Type": "text/calendar; charset=%s" % self._encoding} + return status, headers, body + else: + try: + status, xml_answer = xml_report( + base_prefix, path, xml_content, collection, self._encoding, + lock_stack.close) + except ValueError as e: + logger.warning( + "Bad REPORT request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + headers = {"Content-Type": "text/xml; charset=%s" % self._encoding} + return status, headers, self._xml_response(xml_answer) diff --git a/radicale/item/filter.py b/radicale/item/filter.py index 0b3d01ef9..bc1649655 100644 --- a/radicale/item/filter.py +++ b/radicale/item/filter.py @@ -52,6 +52,27 @@ def date_to_datetime(d: date) -> datetime: return d +def parse_time_range(time_filter: ET.Element) -> Tuple[datetime, datetime]: + start_text = time_filter.get("start") + end_text = time_filter.get("end") + if start_text: + start = datetime.strptime( + start_text, "%Y%m%dT%H%M%SZ").replace( + tzinfo=timezone.utc) + else: + start = DATETIME_MIN + if end_text: + end = datetime.strptime( + end_text, "%Y%m%dT%H%M%SZ").replace( + tzinfo=timezone.utc) + else: + end = DATETIME_MAX + return start, end + +def time_range_timestamps(time_filter: ET.Element) -> Tuple[int, int]: + start, end = parse_time_range(time_filter) + return (math.floor(start.timestamp()), math.ceil(end.timestamp())) + def comp_match(item: "item.Item", filter_: ET.Element, level: int = 0) -> bool: """Check whether the ``item`` matches the comp ``filter_``. @@ -147,21 +168,10 @@ def time_range_match(vobject_item: vobject.base.Component, """Check whether the component/property ``child_name`` of ``vobject_item`` matches the time-range ``filter_``.""" - start_text = filter_.get("start") - end_text = filter_.get("end") - if not start_text and not end_text: + if not filter_.get("start") and not filter_.get("end"): return False - if start_text: - start = datetime.strptime(start_text, "%Y%m%dT%H%M%SZ") - else: - start = datetime.min - if end_text: - end = datetime.strptime(end_text, "%Y%m%dT%H%M%SZ") - else: - end = datetime.max - start = start.replace(tzinfo=timezone.utc) - end = end.replace(tzinfo=timezone.utc) + start, end = parse_time_range(filter_) matched = False def range_fn(range_start: datetime, range_end: datetime, @@ -181,6 +191,34 @@ def infinity_fn(start: datetime) -> bool: return matched +def time_range_fill(vobject_item: vobject.base.Component, + filter_: ET.Element, child_name: str, n: int = 1 + ) -> List[Tuple[datetime, datetime]]: + """Create a list of ``n`` occurances from the component/property ``child_name`` + of ``vobject_item``.""" + if not filter_.get("start") and not filter_.get("end"): + return [] + + start, end = parse_time_range(filter_) + ranges: List[Tuple[datetime, datetime]] = [] + def range_fn(range_start: datetime, range_end: datetime, + is_recurrence: bool) -> bool: + nonlocal ranges + if start < range_end and range_start < end: + ranges.append((range_start, range_end)) + if n > 0 and len(ranges) >= n: + return True + if end < range_start and not is_recurrence: + return True + return False + + def infinity_fn(range_start: datetime) -> bool: + return False + + visit_time_ranges(vobject_item, child_name, range_fn, infinity_fn) + return ranges + + def visit_time_ranges(vobject_item: vobject.base.Component, child_name: str, range_fn: Callable[[datetime, datetime, bool], bool], infinity_fn: Callable[[datetime], bool]) -> None: @@ -543,20 +581,7 @@ def simplify_prefilters(filters: Iterable[ET.Element], collection_tag: str if time_filter.tag != xmlutils.make_clark("C:time-range"): simple = False continue - start_text = time_filter.get("start") - end_text = time_filter.get("end") - if start_text: - start = math.floor(datetime.strptime( - start_text, "%Y%m%dT%H%M%SZ").replace( - tzinfo=timezone.utc).timestamp()) - else: - start = TIMESTAMP_MIN - if end_text: - end = math.ceil(datetime.strptime( - end_text, "%Y%m%dT%H%M%SZ").replace( - tzinfo=timezone.utc).timestamp()) - else: - end = TIMESTAMP_MAX + start, end = time_range_timestamps(time_filter) return tag, start, end, simple return tag, TIMESTAMP_MIN, TIMESTAMP_MAX, simple return None, TIMESTAMP_MIN, TIMESTAMP_MAX, simple diff --git a/radicale/storage/__init__.py b/radicale/storage/__init__.py index 6946f59b4..7822cbb54 100644 --- a/radicale/storage/__init__.py +++ b/radicale/storage/__init__.py @@ -158,6 +158,24 @@ def get_filtered(self, filters: Iterable[ET.Element] continue yield item, simple and (start <= istart or iend <= end) + def get_by_time(self, start: int , end: int + ) -> Iterable["radicale_item.Item"]: + """Fetch all items within a start and end time range. + + Returns a iterable of ``item``s. + + """ + if not self.tag: + return + for item in self.get_all(): + # TODO: Any other component_name here? + if item.component_name not in ("VEVENT",): + continue + istart, iend = item.time_range + if istart >= end or iend <= start: + continue + yield item + def has_uid(self, uid: str) -> bool: """Check if a UID exists in the collection.""" for item in self.get_all(): diff --git a/radicale/tests/__init__.py b/radicale/tests/__init__.py index 63cfda04a..616f62e18 100644 --- a/radicale/tests/__init__.py +++ b/radicale/tests/__init__.py @@ -27,6 +27,7 @@ import tempfile import wsgiref.util import xml.etree.ElementTree as ET +import vobject from io import BytesIO from typing import Any, Dict, List, Optional, Tuple, Union @@ -35,7 +36,7 @@ import radicale from radicale import app, config, types, xmlutils -RESPONSES = Dict[str, Union[int, Dict[str, Tuple[int, ET.Element]]]] +RESPONSES = Dict[str, Union[int, Dict[str, Tuple[int, ET.Element]], vobject.base.Component]] # Enable debug output radicale.log.logger.setLevel(logging.DEBUG) @@ -107,8 +108,7 @@ def start_response(status_: str, headers_: List[Tuple[str, str]] def parse_responses(text: str) -> RESPONSES: xml = DefusedET.fromstring(text) assert xml.tag == xmlutils.make_clark("D:multistatus") - path_responses: Dict[str, Union[ - int, Dict[str, Tuple[int, ET.Element]]]] = {} + path_responses: RESPONSES = {} for response in xml.findall(xmlutils.make_clark("D:response")): href = response.find(xmlutils.make_clark("D:href")) assert href.text not in path_responses @@ -133,6 +133,12 @@ def parse_responses(text: str) -> RESPONSES: path_responses[href.text] = prop_respones return path_responses + @staticmethod + def parse_free_busy(text: str) -> RESPONSES: + path_responses: RESPONSES = {} + path_responses[""] = vobject.readOne(text) + return path_responses + def get(self, path: str, check: Optional[int] = 200, **kwargs ) -> Tuple[int, str]: assert "data" not in kwargs @@ -177,13 +183,18 @@ def proppatch(self, path: str, data: Optional[str] = None, return status, responses def report(self, path: str, data: str, check: Optional[int] = 207, + is_xml: Optional[bool] = True, **kwargs) -> Tuple[int, RESPONSES]: status, _, answer = self.request("REPORT", path, data, check=check, **kwargs) if status < 200 or 300 <= status: return status, {} assert answer is not None - return status, self.parse_responses(answer) + if is_xml: + parsed = self.parse_responses(answer) + else: + parsed = self.parse_free_busy(answer) + return status, parsed def delete(self, path: str, check: Optional[int] = 200, **kwargs ) -> Tuple[int, RESPONSES]: diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index d591773da..ef9e6cf51 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -22,6 +22,7 @@ import os import posixpath +import vobject from typing import Any, Callable, ClassVar, Iterable, List, Optional, Tuple import defusedxml.ElementTree as DefusedET @@ -1360,10 +1361,27 @@ def test_report_item(self) -> None: """) assert len(responses) == 1 response = responses[event_path] - assert not isinstance(response, int) + assert isinstance(response, dict) status, prop = response["D:getetag"] assert status == 200 and prop.text + def test_report_free_busy(self) -> None: + """Test free busy report on a few items""" + calendar_path = "/calendar.ics/" + self.mkcalendar(calendar_path) + for i in (1,2): + filename = "event{}.ics".format(i) + event = get_file_content(filename) + self.put(posixpath.join(calendar_path, filename), event) + code, responses = self.report(calendar_path, """\ + + + +""", 200, is_xml = False) + assert len(responses) == 1 + for response in responses.values(): + assert isinstance(response, vobject.base.Component) + def _report_sync_token( self, calendar_path: str, sync_token: Optional[str] = None ) -> Tuple[str, RESPONSES]: From 4c1d295e813524b3dc6c353a849bdbba3d9dc75b Mon Sep 17 00:00:00 2001 From: Ray Date: Mon, 9 Oct 2023 19:59:04 -0600 Subject: [PATCH 2/8] Fix bug in free busy report serializing a datetime tzinfo --- radicale/app/report.py | 1 + radicale/item/filter.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 1ec32b068..72c0d0f08 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -65,6 +65,7 @@ def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Eleme vfb.add('dtstart').value, vfb.add('dtend').value = occurrence return (client.OK, cal.serialize()) + def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], collection: storage.BaseCollection, encoding: str, unlock_storage_fn: Callable[[], None] diff --git a/radicale/item/filter.py b/radicale/item/filter.py index bc1649655..af3b0d19e 100644 --- a/radicale/item/filter.py +++ b/radicale/item/filter.py @@ -48,7 +48,8 @@ def date_to_datetime(d: date) -> datetime: if not isinstance(d, datetime): d = datetime.combine(d, datetime.min.time()) if not d.tzinfo: - d = d.replace(tzinfo=timezone.utc) + # NOTE: using vobject's UTC as it wasn't playing well with datetime's. + d = d.replace(tzinfo=vobject.icalendar.utc) return d From b0f131cac287c8a35d3b97f535d6e217cb6fed24 Mon Sep 17 00:00:00 2001 From: Ray Date: Wed, 11 Oct 2023 12:09:11 -0600 Subject: [PATCH 3/8] Improve free-busy report --- radicale/app/report.py | 77 +++++++++++++++++++++++++++++++----- radicale/config.py | 9 ++++- radicale/storage/__init__.py | 18 --------- radicale/tests/test_base.py | 6 ++- 4 files changed, 80 insertions(+), 30 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 72c0d0f08..c8789d339 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -33,15 +33,16 @@ from vobject.base import ContentLine import radicale.item as radicale_item -from radicale import httputils, pathutils, storage, types, xmlutils +from radicale import httputils, pathutils, storage, types, xmlutils, config from radicale.app.base import Access, ApplicationBase from radicale.item import filter as radicale_filter from radicale.log import logger def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], - collection: storage.BaseCollection, encoding: str, - unlock_storage_fn: Callable[[], None] - ) -> Tuple[int, str]: + collection: storage.BaseCollection, encoding: str, + unlock_storage_fn: Callable[[], None], + max_occurrence: int + ) -> Tuple[int, str]: multistatus = ET.Element(xmlutils.make_clark("D:multistatus")) if xml_request is None: return client.MULTI_STATUS, multistatus @@ -53,16 +54,73 @@ def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Eleme return client.FORBIDDEN, xmlutils.webdav_error("D:supported-report") time_range_element = root.find(xmlutils.make_clark("C:time-range")) - start,end = radicale_filter.time_range_timestamps(time_range_element) - items = list(collection.get_by_time(start, end)) + + # Build a single filter from the free busy query for retrieval + # TODO: filter for VFREEBUSY in additional to VEVENT but + # test_filter doesn't support that yet. + vevent_cf_element = ET.Element(xmlutils.make_clark("C:comp-filter"), + attrib={'name':'VEVENT'}) + vevent_cf_element.append(time_range_element) + vcalendar_cf_element = ET.Element(xmlutils.make_clark("C:comp-filter"), + attrib={'name':'VCALENDAR'}) + vcalendar_cf_element.append(vevent_cf_element) + filter_element = ET.Element(xmlutils.make_clark("C:filter")) + filter_element.append(vcalendar_cf_element) + filters = (filter_element,) + + # First pull from storage + retrieved_items = list(collection.get_filtered(filters)) + # !!! Don't access storage after this !!! + unlock_storage_fn() cal = vobject.iCalendar() - for item in items: - occurrences = radicale_filter.time_range_fill(item.vobject_item, time_range_element, "VEVENT") + collection_tag = collection.tag + while retrieved_items: + # Second filtering before evaluating occurrences. + # ``item.vobject_item`` might be accessed during filtering. + # Don't keep reference to ``item``, because VObject requires a lot of + # memory. + item, filter_matched = retrieved_items.pop(0) + if not filter_matched: + try: + if not test_filter(collection_tag, item, filter_element): + continue + except ValueError as e: + raise ValueError("Failed to free-busy filter item %r from %r: %s" % + (item.href, collection.path, e)) from e + except Exception as e: + raise RuntimeError("Failed to free-busy filter item %r from %r: %s" % + (item.href, collection.path, e)) from e + + fbtype = None + if item.component_name == 'VEVENT': + transp = getattr(item.vobject_item, 'transp', None) + if transp and transp.value != 'OPAQUE': + continue + + status = getattr(item.vobject_item, 'status', None) + if not status or status.value == 'CONFIRMED': + fbtype = 'BUSY' + elif status.value == 'CANCELLED': + fbtype = 'FREE' + elif status.value == 'TENTATIVE': + fbtype = 'BUSY-TENTATIVE' + else: + # Could do fbtype = status.value for x-name, I prefer this + fbtype = 'BUSY' + + # TODO: coalesce overlapping periods + + occurrences = radicale_filter.time_range_fill(item.vobject_item, + time_range_element, + "VEVENT", + n=max_occurrence) for occurrence in occurrences: vfb = cal.add('vfreebusy') vfb.add('dtstamp').value = item.vobject_item.vevent.dtstamp.value vfb.add('dtstart').value, vfb.add('dtend').value = occurrence + if fbtype: + vfb.add('fbtype').value = fbtype return (client.OK, cal.serialize()) @@ -457,10 +515,11 @@ def do_REPORT(self, environ: types.WSGIEnviron, base_prefix: str, if xml_content is not None and \ xml_content.tag == xmlutils.make_clark("C:free-busy-query"): + max_occurrence = self.configuration.get("reporting", "max_freebusy_occurrence") try: status, body = free_busy_report( base_prefix, path, xml_content, collection, self._encoding, - lock_stack.close) + lock_stack.close, max_occurrence) except ValueError as e: logger.warning( "Bad REPORT request on %r: %s", path, e, exc_info=True) diff --git a/radicale/config.py b/radicale/config.py index 967580cb0..10b36a6e9 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -293,8 +293,13 @@ def json_str(value: Any) -> dict: "help": "mask passwords in logs", "type": bool})])), ("headers", OrderedDict([ - ("_allow_extra", str)]))]) - + ("_allow_extra", str)])), + ("reporting", OrderedDict([ + ("max_freebusy_occurrence", { + "value": "10000", + "help": "number of occurrences per event when reporting", + "type": positive_int})])) + ]) def parse_compound_paths(*compound_paths: Optional[str] ) -> List[Tuple[str, bool]]: diff --git a/radicale/storage/__init__.py b/radicale/storage/__init__.py index 7822cbb54..6946f59b4 100644 --- a/radicale/storage/__init__.py +++ b/radicale/storage/__init__.py @@ -158,24 +158,6 @@ def get_filtered(self, filters: Iterable[ET.Element] continue yield item, simple and (start <= istart or iend <= end) - def get_by_time(self, start: int , end: int - ) -> Iterable["radicale_item.Item"]: - """Fetch all items within a start and end time range. - - Returns a iterable of ``item``s. - - """ - if not self.tag: - return - for item in self.get_all(): - # TODO: Any other component_name here? - if item.component_name not in ("VEVENT",): - continue - istart, iend = item.time_range - if istart >= end or iend <= start: - continue - yield item - def has_uid(self, uid: str) -> bool: """Check if a UID exists in the collection.""" for item in self.get_all(): diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index ef9e6cf51..94fca05ea 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -1378,9 +1378,13 @@ def test_report_free_busy(self) -> None: """, 200, is_xml = False) - assert len(responses) == 1 for response in responses.values(): assert isinstance(response, vobject.base.Component) + assert len(responses) == 1 + vcalendar = list(responses.values())[0] + assert len(vcalendar.vfreebusy_list) == 2 + for vfb in vcalendar.vfreebusy_list: + assert vfb.fbtype.value == 'BUSY' def _report_sync_token( self, calendar_path: str, sync_token: Optional[str] = None From 204623d6560d6155cfcbb2f16d253d8436e16ffe Mon Sep 17 00:00:00 2001 From: Ray Date: Wed, 11 Oct 2023 12:41:50 -0600 Subject: [PATCH 4/8] Package housekeeping --- CHANGELOG.md | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38ffafa3e..42a159fea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,13 @@ * Dependency: limit typegard version < 3 * General: code cosmetics +## 3.2.0.a1 + +* Added free-busy report +* Using icalendar's tzinfo on created datetime fix issue with icalendar +* Added 'max_freebusy_occurrences` setting to avoid potential DOS on reports +* Refactored some date parsing code + ## 3.1.8 * Fix setuptools requirement if installing wheel diff --git a/setup.py b/setup.py index f5d906a2c..7aa5d391a 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.dev" +VERSION = "3.2.0.a1" with open("README.md", encoding="utf-8") as f: long_description = f.read() From 29b7cd8d54f3c7e098d83415211b1fc9b35b6380 Mon Sep 17 00:00:00 2001 From: Ray Date: Sat, 9 Dec 2023 18:22:03 -0700 Subject: [PATCH 5/8] Fix for free-busy `fbtype` statuses --- CHANGELOG.md | 4 ++++ radicale/app/report.py | 4 ++-- radicale/tests/static/event10.ics | 36 +++++++++++++++++++++++++++++++ radicale/tests/test_base.py | 10 ++++++--- setup.py | 2 +- 5 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 radicale/tests/static/event10.ics diff --git a/CHANGELOG.md b/CHANGELOG.md index 42a159fea..b6ae0ad9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,10 @@ * Dependency: limit typegard version < 3 * General: code cosmetics +## 3.2.0.a2 + +* Fix for free-busy `fbtype` statuses + ## 3.2.0.a1 * Added free-busy report diff --git a/radicale/app/report.py b/radicale/app/report.py index c8789d339..ef0adc2f3 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -94,11 +94,11 @@ def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Eleme fbtype = None if item.component_name == 'VEVENT': - transp = getattr(item.vobject_item, 'transp', None) + transp = getattr(item.vobject_item.vevent, 'transp', None) if transp and transp.value != 'OPAQUE': continue - status = getattr(item.vobject_item, 'status', None) + status = getattr(item.vobject_item.vevent, 'status', None) if not status or status.value == 'CONFIRMED': fbtype = 'BUSY' elif status.value == 'CANCELLED': diff --git a/radicale/tests/static/event10.ics b/radicale/tests/static/event10.ics new file mode 100644 index 000000000..3faa034d2 --- /dev/null +++ b/radicale/tests/static/event10.ics @@ -0,0 +1,36 @@ +BEGIN:VCALENDAR +PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN +VERSION:2.0 +BEGIN:VTIMEZONE +TZID:Europe/Paris +X-LIC-LOCATION:Europe/Paris +BEGIN:DAYLIGHT +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:CEST +DTSTART:19700329T020000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:CET +DTSTART:19701025T030000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +CREATED:20130902T150157Z +LAST-MODIFIED:20130902T150158Z +DTSTAMP:20130902T150158Z +UID:event10 +SUMMARY:Event +CATEGORIES:some_category1,another_category2 +ORGANIZER:mailto:unclesam@example.com +ATTENDEE;ROLE=REQ-PARTICIPANT;PARTSTAT=TENTATIVE;CN=Jane Doe:MAILTO:janedoe@example.com +ATTENDEE;ROLE=REQ-PARTICIPANT;DELEGATED-FROM="MAILTO:bob@host.com";PARTSTAT=ACCEPTED;CN=John Doe:MAILTO:johndoe@example.com +DTSTART;TZID=Europe/Paris:20130901T180000 +DTEND;TZID=Europe/Paris:20130901T190000 +STATUS:CANCELLED +END:VEVENT +END:VCALENDAR diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index 94fca05ea..479893f65 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -1369,7 +1369,7 @@ def test_report_free_busy(self) -> None: """Test free busy report on a few items""" calendar_path = "/calendar.ics/" self.mkcalendar(calendar_path) - for i in (1,2): + for i in (1,2,10): filename = "event{}.ics".format(i) event = get_file_content(filename) self.put(posixpath.join(calendar_path, filename), event) @@ -1382,9 +1382,13 @@ def test_report_free_busy(self) -> None: assert isinstance(response, vobject.base.Component) assert len(responses) == 1 vcalendar = list(responses.values())[0] - assert len(vcalendar.vfreebusy_list) == 2 + assert len(vcalendar.vfreebusy_list) == 3 + types = {} for vfb in vcalendar.vfreebusy_list: - assert vfb.fbtype.value == 'BUSY' + if vfb.fbtype.value not in types: + types[vfb.fbtype.value] = 0 + types[vfb.fbtype.value] += 1 + assert types == {'BUSY':2, 'FREE':1} def _report_sync_token( self, calendar_path: str, sync_token: Optional[str] = None diff --git a/setup.py b/setup.py index 7aa5d391a..cc3b1c25c 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.2.0.a1" +VERSION = "3.2.0.a2" with open("README.md", encoding="utf-8") as f: long_description = f.read() From d6c0a05771b6d91bf8625235200a78a116ea15a4 Mon Sep 17 00:00:00 2001 From: Ray Date: Wed, 14 Aug 2024 11:15:30 -0600 Subject: [PATCH 6/8] Style fixes for tox linting --- radicale/app/report.py | 18 ++++++++++++------ radicale/config.py | 1 + radicale/item/filter.py | 3 +++ radicale/tests/__init__.py | 2 +- radicale/tests/test_base.py | 16 +++++++++------- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index ef0adc2f3..9bca7f45a 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -23,26 +23,31 @@ import posixpath import socket import xml.etree.ElementTree as ET -import vobject from http import client from typing import (Any, Callable, Iterable, Iterator, List, Optional, Sequence, Tuple, Union) from urllib.parse import unquote, urlparse +import vobject import vobject.base from vobject.base import ContentLine import radicale.item as radicale_item -from radicale import httputils, pathutils, storage, types, xmlutils, config +from radicale import httputils, pathutils, storage, types, xmlutils from radicale.app.base import Access, ApplicationBase from radicale.item import filter as radicale_filter from radicale.log import logger + def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], collection: storage.BaseCollection, encoding: str, unlock_storage_fn: Callable[[], None], max_occurrence: int - ) -> Tuple[int, str]: + ) -> Tuple[int, Union[ET.Element, str]]: + # NOTE: this function returns both an Element and a string because + # free-busy reports are an edge-case on the return type according + # to the spec. + multistatus = ET.Element(xmlutils.make_clark("D:multistatus")) if xml_request is None: return client.MULTI_STATUS, multistatus @@ -54,15 +59,16 @@ def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Eleme return client.FORBIDDEN, xmlutils.webdav_error("D:supported-report") time_range_element = root.find(xmlutils.make_clark("C:time-range")) + assert isinstance(time_range_element, ET.Element) # Build a single filter from the free busy query for retrieval # TODO: filter for VFREEBUSY in additional to VEVENT but # test_filter doesn't support that yet. vevent_cf_element = ET.Element(xmlutils.make_clark("C:comp-filter"), - attrib={'name':'VEVENT'}) + attrib={'name': 'VEVENT'}) vevent_cf_element.append(time_range_element) vcalendar_cf_element = ET.Element(xmlutils.make_clark("C:comp-filter"), - attrib={'name':'VCALENDAR'}) + attrib={'name': 'VCALENDAR'}) vcalendar_cf_element.append(vevent_cf_element) filter_element = ET.Element(xmlutils.make_clark("C:filter")) filter_element.append(vcalendar_cf_element) @@ -525,7 +531,7 @@ def do_REPORT(self, environ: types.WSGIEnviron, base_prefix: str, "Bad REPORT request on %r: %s", path, e, exc_info=True) return httputils.BAD_REQUEST headers = {"Content-Type": "text/calendar; charset=%s" % self._encoding} - return status, headers, body + return status, headers, str(body) else: try: status, xml_answer = xml_report( diff --git a/radicale/config.py b/radicale/config.py index 10b36a6e9..b6ab138d0 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -301,6 +301,7 @@ def json_str(value: Any) -> dict: "type": positive_int})])) ]) + def parse_compound_paths(*compound_paths: Optional[str] ) -> List[Tuple[str, bool]]: """Parse a compound path and return the individual paths. diff --git a/radicale/item/filter.py b/radicale/item/filter.py index af3b0d19e..ef6db2f57 100644 --- a/radicale/item/filter.py +++ b/radicale/item/filter.py @@ -70,10 +70,12 @@ def parse_time_range(time_filter: ET.Element) -> Tuple[datetime, datetime]: end = DATETIME_MAX return start, end + def time_range_timestamps(time_filter: ET.Element) -> Tuple[int, int]: start, end = parse_time_range(time_filter) return (math.floor(start.timestamp()), math.ceil(end.timestamp())) + def comp_match(item: "item.Item", filter_: ET.Element, level: int = 0) -> bool: """Check whether the ``item`` matches the comp ``filter_``. @@ -202,6 +204,7 @@ def time_range_fill(vobject_item: vobject.base.Component, start, end = parse_time_range(filter_) ranges: List[Tuple[datetime, datetime]] = [] + def range_fn(range_start: datetime, range_end: datetime, is_recurrence: bool) -> bool: nonlocal ranges diff --git a/radicale/tests/__init__.py b/radicale/tests/__init__.py index 616f62e18..f335fd3b4 100644 --- a/radicale/tests/__init__.py +++ b/radicale/tests/__init__.py @@ -27,11 +27,11 @@ import tempfile import wsgiref.util import xml.etree.ElementTree as ET -import vobject from io import BytesIO from typing import Any, Dict, List, Optional, Tuple, Union import defusedxml.ElementTree as DefusedET +import vobject import radicale from radicale import app, config, types, xmlutils diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index 479893f65..b7edb5866 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -22,10 +22,10 @@ import os import posixpath -import vobject from typing import Any, Callable, ClassVar, Iterable, List, Optional, Tuple import defusedxml.ElementTree as DefusedET +import vobject from radicale import storage, xmlutils from radicale.tests import RESPONSES, BaseTest @@ -1369,7 +1369,7 @@ def test_report_free_busy(self) -> None: """Test free busy report on a few items""" calendar_path = "/calendar.ics/" self.mkcalendar(calendar_path) - for i in (1,2,10): + for i in (1, 2, 10): filename = "event{}.ics".format(i) event = get_file_content(filename) self.put(posixpath.join(calendar_path, filename), event) @@ -1377,18 +1377,20 @@ def test_report_free_busy(self) -> None: -""", 200, is_xml = False) +""", 200, is_xml=False) for response in responses.values(): assert isinstance(response, vobject.base.Component) assert len(responses) == 1 vcalendar = list(responses.values())[0] + assert isinstance(vcalendar, vobject.base.Component) assert len(vcalendar.vfreebusy_list) == 3 types = {} for vfb in vcalendar.vfreebusy_list: - if vfb.fbtype.value not in types: - types[vfb.fbtype.value] = 0 - types[vfb.fbtype.value] += 1 - assert types == {'BUSY':2, 'FREE':1} + fbtype_val = vfb.fbtype.value + if fbtype_val not in types: + types[fbtype_val] = 0 + types[fbtype_val] += 1 + assert types == {'BUSY': 2, 'FREE': 1} def _report_sync_token( self, calendar_path: str, sync_token: Optional[str] = None From 906d391fe35f66f9c28008a5a56fbb96d63181db Mon Sep 17 00:00:00 2001 From: Ray Date: Thu, 15 Aug 2024 15:03:31 -0600 Subject: [PATCH 7/8] Drop VERSION change and move changelog inline with releases --- CHANGELOG.md | 15 ++++----------- setup.py | 2 +- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6ae0ad9c..df11dd965 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog ## 3.dev +* Fix: Using icalendar's tzinfo on created datetime to fix issue with icalendar +* Enhancement: Added free-busy report +* Enhancement: Added 'max_freebusy_occurrences` setting to avoid potential DOS on reports +* Improve: Refactored some date parsing code ## 3.2.2 * Enhancement: add support for auth.type=denyall (will be default for security reasons in upcoming releases) @@ -53,17 +57,6 @@ * Dependency: limit typegard version < 3 * General: code cosmetics -## 3.2.0.a2 - -* Fix for free-busy `fbtype` statuses - -## 3.2.0.a1 - -* Added free-busy report -* Using icalendar's tzinfo on created datetime fix issue with icalendar -* Added 'max_freebusy_occurrences` setting to avoid potential DOS on reports -* Refactored some date parsing code - ## 3.1.8 * Fix setuptools requirement if installing wheel diff --git a/setup.py b/setup.py index cc3b1c25c..f5d906a2c 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.2.0.a2" +VERSION = "3.dev" with open("README.md", encoding="utf-8") as f: long_description = f.read() From 408a03a3c0e73feacfa4ae5a3d4a1e69eda07c42 Mon Sep 17 00:00:00 2001 From: Ray Date: Fri, 16 Aug 2024 14:57:30 -0600 Subject: [PATCH 8/8] Better freebusy occurrence limit handling and added documentation --- DOCUMENTATION.md | 12 ++++++++++++ config | 6 ++++++ radicale/app/report.py | 10 +++++++++- radicale/tests/test_base.py | 8 ++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index c012b6671..15f54d0be 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1023,6 +1023,18 @@ RabbitMQ queue type for the topic. Default: classic +#### reporting +##### max_freebusy_occurrence + +When returning a free-busy report, a list of busy time occurrences are +generated based on a given time frame. Large time frames could +generate a lot of occurrences based on the time frame supplied. This +setting limits the lookup to prevent potential denial of service +attacks on large time frames. If the limit is reached, an HTTP error +is thrown instead of returning the results. + +Default: 10000 + ## Supported Clients Radicale has been tested with: diff --git a/config b/config index a9fe9da71..829ad7db7 100644 --- a/config +++ b/config @@ -172,3 +172,9 @@ #rabbitmq_endpoint = #rabbitmq_topic = #rabbitmq_queue_type = classic + +[reporting] + +# When returning a free-busy report, limit the number of returned +# occurences per event to prevent DOS attacks. +#max_freebusy_occurrence = 10000 \ No newline at end of file diff --git a/radicale/app/report.py b/radicale/app/report.py index 7ea34d214..9d57b3895 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -117,10 +117,18 @@ def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Eleme # TODO: coalesce overlapping periods + if max_occurrence > 0: + n_occurrences = max_occurrence+1 + else: + n_occurrences = 0 occurrences = radicale_filter.time_range_fill(item.vobject_item, time_range_element, "VEVENT", - n=max_occurrence) + n=n_occurrences) + if len(occurrences) >= max_occurrence: + raise ValueError("FREEBUSY occurrences limit of {} hit" + .format(max_occurrence)) + for occurrence in occurrences: vfb = cal.add('vfreebusy') vfb.add('dtstamp').value = item.vobject_item.vevent.dtstamp.value diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index ae3669d0d..fc708ebc0 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -1392,6 +1392,14 @@ def test_report_free_busy(self) -> None: types[fbtype_val] += 1 assert types == {'BUSY': 2, 'FREE': 1} + # Test max_freebusy_occurrence limit + self.configure({"reporting": {"max_freebusy_occurrence": 1}}) + code, responses = self.report(calendar_path, """\ + + + +""", 400, is_xml=False) + def _report_sync_token( self, calendar_path: str, sync_token: Optional[str] = None ) -> Tuple[str, RESPONSES]: