Buffered io merge into master

packaging/rhel/xrootd-ceph-buffered.spec.in

@@ -0,0 +1,179 @@
+#-------------------------------------------------------------------------------
+# Helper macros
+#-------------------------------------------------------------------------------
+%if %{?rhel:1}%{!?rhel:0}
+    %if %{rhel} >= 7
+        %define use_systemd 1
+    %else
+        %define use_systemd 0
+    %endif
+%else
+    %if %{?fedora}%{!?fedora:0} >= 19
+        %define use_systemd 1
+    %else
+        %define use_systemd 0
+    %endif
+%endif
+
+%if %{?fedora}%{!?fedora:0} >= 22
+    %define use_libc_semaphore 1
+%else
+    %define use_libc_semaphore 0
+%endif
+
+%if %{?_with_ceph11:1}%{!?_with_ceph11:0}
+    %define _with_ceph 1
+%endif
+
+%if %{?rhel:1}%{!?rhel:0}
+    %if %{rhel} > 7
+        %define use_cmake3 0
+    %else
+        %define use_cmake3 1
+    %endif
+%else
+    %define use_cmake3 0
+%endif
+
+#-------------------------------------------------------------------------------
+# Package definitions
+#-------------------------------------------------------------------------------
+Name:      xrootd-ceph-buffered
+Epoch:     1
+Version:   __VERSION__
+Release:   __RELEASE__%{?dist}%{?_with_clang:.clang}
+Summary:   CEPH plug-in for XRootD
+Group:     System Environment/Daemons
+License:   LGPLv3+
+URL:       http://xrootd.org/
+
+# git clone http://xrootd.org/repo/xrootd.git xrootd
+# cd xrootd
+# git-archive master | gzip -9 > ~/rpmbuild/SOURCES/xrootd.tgz
+Source0:   xrootd-ceph-buffered.tar.gz
+
+BuildRoot: %{_tmppath}/%{name}-root
+
+%if %{use_cmake3}
+BuildRequires: cmake3
+%else
+BuildRequires: cmake
+%endif
+
+%if %{?_with_tests:1}%{!?_with_tests:0}
+BuildRequires: cppunit-devel
+%endif
+
+BuildRequires: librados-devel = 2:14.2.22
+BuildRequires: libradosstriper-devel = 2:14.2.22
+
+%if %{?_with_clang:1}%{!?_with_clang:0}
+BuildRequires: clang
+%endif
+
+#BuildRequires: xrootd-server-devel%{?_isa} = %{epoch}:%{version}-%{release}
+#BuildRequires: xrootd-private-devel%{?_isa} = %{epoch}:%{version}-%{release}
+#BuildRequires: xrootd-libs%{?_isa} = %{epoch}:%{version}-%{release}
+#BuildRequires: xrootd-server-libs%{?_isa} = %{epoch}:%{version}-%{release}
+#BuildRequires: xrootd-client-libs%{?_isa} = %{epoch}:%{version}-%{release}
+
+#Requires: xrootd-server-libs%{?_isa} = %{epoch}:%{version}-%{release}
+#Requires: xrootd-client-libs%{?_isa} = %{epoch}:%{version}-%{release}
+#Requires: xrootd-libs%{?_isa} = %{epoch}:%{version}-%{release}
+
+BuildRequires: xrootd-server-devel%{?_isa} >= 1:5.3.3
+BuildRequires: xrootd-private-devel%{?_isa} >= 1:5.3.3
+BuildRequires: xrootd-libs%{?_isa} >= 1:5.3.1
+BuildRequires: xrootd-server-libs%{?_isa} >= 1:5.3.3
+BuildRequires: xrootd-client-libs%{?_isa} >= 1:5.3.3
+
+Requires: xrootd-server-libs%{?_isa} >= 1:5.3.3
+Requires: xrootd-client-libs%{?_isa} >= 1:5.3.3
+Requires: xrootd-libs%{?_isa} >= 1:5.3.3
+
+%description
+The xrootd-ceph-buffered is an OSS layer plug-in for the XRootD server for interfacing
+with the Ceph storage platform.
+
+#-------------------------------------------------------------------------------
+# Build instructions
+#-------------------------------------------------------------------------------
+%prep
+%setup -c -n xrootd-ceph-buffered
+
+%build
+cd xrootd-ceph-buffered
+
+%if %{?_with_clang:1}%{!?_with_clang:0}
+export CC=clang
+export CXX=clang++
+%endif
+
+mkdir build
+pushd build
+
+%if %{use_cmake3}
+cmake3 \
+%else
+cmake  \
+%endif
+       -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+%if %{?_with_tests:1}%{!?_with_tests:0}
+       -DENABLE_TESTS=TRUE \
+%else
+       -DENABLE_TESTS=FALSE \
+%endif
+      ../
+
+make -i VERBOSE=1 %{?_smp_mflags}
+popd
+
+#-------------------------------------------------------------------------------
+# Installation
+#-------------------------------------------------------------------------------
+%install
+rm -rf $RPM_BUILD_ROOT
+
+#-------------------------------------------------------------------------------
+# Install 4.x.y
+#-------------------------------------------------------------------------------
+pushd xrootd-ceph-buffered
+pushd  build
+make install DESTDIR=$RPM_BUILD_ROOT
+popd
+
+# ceph posix unversioned so
+rm -f $RPM_BUILD_ROOT%{_libdir}/libXrdCephPosix.so
+
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+#-------------------------------------------------------------------------------
+# Files
+#-------------------------------------------------------------------------------
+%files
+%defattr(-,root,root,-)
+%{_libdir}/libXrdCeph-5.so
+%{_libdir}/libXrdCephXattr-5.so
+%{_libdir}/libXrdCephPosix.so*
+
+%if %{?_with_tests:1}%{!?_with_tests:0}
+%files tests
+%defattr(-,root,root,-)
+%{_libdir}/libXrdCephTests*.so
+%endif
+
+#-------------------------------------------------------------------------------
+# Changelog
+#-------------------------------------------------------------------------------
+%changelog
+* Mon Mar 14 2022 Jyothish Thomas <[email protected]>
+-offline file bug fix
+* Wed Dec 16 2020 George Patargias <[email protected]>
+- updated version for librados-devel and libradosstriper-devel to 14.2.15 following the recent upgrade on external Echo gateways
+- fixed version in xrootd-ceph-buffered shared libraries
+* Mon Mar 02 2020 Michal Simon <[email protected]>
+- fixed RPM dependencies
+* Thu Mar 08 2018 Michal Simon <[email protected]>
+- initial release

src/XrdCeph.cmake

@@ -46,7 +46,18 @@ add_library(
   XrdCeph/XrdCephOss.cc       XrdCeph/XrdCephOss.hh
   XrdCeph/XrdCephOssFile.cc   XrdCeph/XrdCephOssFile.hh
   XrdCeph/XrdCephOssDir.cc    XrdCeph/XrdCephOssDir.hh
-  XrdCeph/XrdCephBulkAioRead.cc XrdCeph/XrdCephBulkAioRead.hh)
+  XrdCeph/XrdCephBulkAioRead.cc XrdCeph/XrdCephBulkAioRead.hh
+  XrdCeph/XrdCephOssBufferedFile.cc   XrdCeph/XrdCephOssBufferedFile.hh
+  XrdCeph/XrdCephOssReadVFile.cc   XrdCeph/XrdCephOssReadVFile.hh
+  XrdCeph/XrdCephBuffers/XrdCephBufferDataSimple.cc XrdCeph/XrdCephBuffers/XrdCephBufferDataSimple.hh
+  XrdCeph/XrdCephBuffers/XrdCephBufferAlgSimple.cc  XrdCeph/XrdCephBuffers/XrdCephBufferAlgSimple.hh 
+  XrdCeph/XrdCephBuffers/CephIOAdapterRaw.cc  XrdCeph/XrdCephBuffers/CephIOAdapterRaw.hh
+  XrdCeph/XrdCephBuffers/CephIOAdapterAIORaw.cc  XrdCeph/XrdCephBuffers/CephIOAdapterAIORaw.hh
+  XrdCeph/XrdCephBuffers/BufferUtils.cc  XrdCeph/XrdCephBuffers/BufferUtils.hh
+  XrdCeph/XrdCephBuffers/XrdCephReadVNoOp.cc  XrdCeph/XrdCephBuffers/XrdCephReadVNoOp.hh
+  XrdCeph/XrdCephBuffers/XrdCephReadVBasic.cc  XrdCeph/XrdCephBuffers/XrdCephReadVBasic.hh
+)
+
 
 target_link_libraries(
   ${LIB_XRD_CEPH}

src/XrdCeph/XrdCephBuffers/BufferUtils.cc

@@ -0,0 +1,169 @@
+
+#include "BufferUtils.hh"
+#include <algorithm> // std::max
+
+using namespace XrdCephBuffer;
+
+#ifdef CEPHBUFDEBUG
+// to synchronise logging statements 
+  std::mutex cephbuf_iolock;
+#endif 
+
+// ------------------------------------------------------ //
+//         Extent       //
+
+bool Extent::in_extent(off_t pos) const
+{
+    return ((pos > begin()) && (pos < end()));
+}
+
+bool Extent::isContiguous(const Extent &rhs) const
+{
+    // does the rhs connect directly to the end of the first
+    if (end() != rhs.begin())
+        return false;
+    return true;
+}
+
+bool Extent::allInExtent(off_t pos, size_t len) const
+{
+    // is all the range in this extent
+    if ((pos < begin()) || (pos >= end()))
+        return false;
+
+    if (off_t(pos + len) > end())
+        return false;
+    return true;
+}
+bool Extent::someInExtent(off_t pos, size_t len) const
+{ // is some of the range in this extent
+    if ((off_t(pos + len) < begin()) || (pos >= end()))
+        return false;
+    return true;
+}
+
+Extent Extent::containedExtent(off_t pos, size_t len) const
+{
+    // return the subset of input range that is in this extent
+    off_t subbeg = std::max(begin(), pos);
+    off_t subend = std::min(end(), off_t(pos + len));
+
+    return Extent(subbeg, subend - subbeg);
+}
+Extent Extent::containedExtent(const Extent &rhs) const
+{
+    return containedExtent(rhs.begin(), rhs.len());
+}
+
+bool Extent::operator<(const Extent &rhs) const
+{
+    // comparison primarily on begin values
+    // use end values if begin values are equal.
+
+    if (begin() > rhs.begin()) return false;
+    if (begin() < rhs.begin()) return true;
+    if (end()   < rhs.end() )  return true;
+    return false; 
+}
+bool Extent::operator==(const Extent &rhs) const
+{
+    // equivalence based only on start and end
+    if (begin() != rhs.begin())
+        return false;
+    if (end() != rhs.end())
+        return false;
+    return true;
+}
+
+// ------------------------------------------------------ //
+//         ExtentHolder       //
+
+ExtentHolder::ExtentHolder() {}
+
+ExtentHolder::ExtentHolder(size_t elements)
+{
+    m_extents.reserve(elements);
+}
+
+ExtentHolder::ExtentHolder(const ExtentContainer &extents)
+{
+    m_extents.reserve(extents.size());
+    for (ExtentContainer::const_iterator vit = m_extents.cbegin(); vit != m_extents.cend(); ++vit) {
+        push_back(*vit);
+    }
+
+}
+ExtentHolder::~ExtentHolder()
+{
+    m_extents.clear();
+}
+
+void ExtentHolder::push_back(const Extent & in) {
+    if (size()) {
+        m_begin = std::min(m_begin, in.begin());
+        m_end   = std::max(m_end, in.end());
+    } else {
+        m_begin = in.begin();
+        m_end = in.end();
+    }
+    return m_extents.push_back(in);
+}
+
+
+
+Extent ExtentHolder::asExtent() const {
+    // if (!size()) return Extent(0,0);
+    // ExtentContainer se = getSortedExtents();
+    // off_t b = se.front().begin();
+    // off_t e = se.back().end();
+
+    return Extent(m_begin, m_end-m_begin);
+
+}
+
+size_t ExtentHolder::bytesContained() const {
+    size_t nbytes{0};
+    for (ExtentContainer::const_iterator vit = m_extents.cbegin(); vit != m_extents.cend(); ++vit) {
+        nbytes += vit->len();
+    }
+    return nbytes;
+}
+
+size_t ExtentHolder::bytesMissing() const {
+    size_t bytesUsed = bytesContained();
+    size_t totalRange = asExtent().len(); //might be expensive to call
+    return totalRange - bytesUsed;
+}   
+
+
+void ExtentHolder::sort() {
+        std::sort(m_extents.begin(), m_extents.end());
+}
+
+
+ExtentContainer ExtentHolder::getSortedExtents() const {
+    ExtentContainer v;
+    v.assign(m_extents.begin(), m_extents.end() );
+    std::sort(v.begin(), v.end());
+    return v;
+}
+
+ExtentContainer ExtentHolder::getExtents() const {
+    ExtentContainer v;
+    v.assign(m_extents.begin(), m_extents.end() );
+    return v;
+}
+
+// ------------------------------------------------------ //
+//         Timer ns       //
+
+Timer_ns::Timer_ns(long &output) : m_output_val(output)
+{
+    m_start = std::chrono::steady_clock::now();
+}
+
+Timer_ns::~Timer_ns()
+{
+    auto end = std::chrono::steady_clock::now();
+    m_output_val = std::chrono::duration_cast<std::chrono::nanoseconds>(end - m_start).count();
+}