Merge pull request #1733 from farialima/fix-57 by farialima

Inclusion from LDAP data sources supports RFC 2696 Paged Results control (#57)
sympa-community · Nov 29, 2023 · 2711129 · 2711129
2 parents ad7107b + c428181
commit 2711129
Show file tree

Hide file tree

Showing 3 changed files with 70 additions and 10 deletions.
diff --git a/src/lib/Sympa/Config/Schema.pm b/src/lib/Sympa/Config/Schema.pm
@@ -3466,7 +3466,16 @@ our %pinfo = (
                 format_s   => '$time_ranges',
                 occurrence => '0-1',
                 not_before => '6.2a.16',
-            }
+            },
+            pagesize => {
+                context      => [qw(list)],
+                order        => 12,
+                gettext_comment =>
+                    'Number of records to fetch per batch (paging), for a LDAP server that supports paging. If not set or set to zero, do not use paging. Typically 1000 for an Active Directory server, to avoid "SizeLimit" errors',
+                gettext_id   => "Page size",
+                format       => '\d*',
+                length       => 6,
+            },
         },
         occurrence => '0-n'
     },
@@ -3691,7 +3700,16 @@ our %pinfo = (
                 format_s   => '$time_ranges',
                 occurrence => '0-1',
                 not_before => '6.2a.16',
-            }
+            },
+            pagesize => {
+                context      => [qw(list)],
+                order        => 12,
+                gettext_comment =>
+                    'Number of records to fetch per batch (paging), for a LDAP server that supports paging. If not set or set to zero, do not use paging. Typically 1000 for an Active Directory server, to avoid "SizeLimit" errors',
+                gettext_id   => "Page size",
+                format       => '\d*',
+                length       => 6,
+            },
         },
         occurrence => '0-n'
     },

diff --git a/src/lib/Sympa/DataSource/LDAP.pm b/src/lib/Sympa/DataSource/LDAP.pm
@@ -25,6 +25,8 @@ package Sympa::DataSource::LDAP;
 use strict;
 use warnings;
 
+use Net::LDAP::Control::Paged;
+
 use Sympa::Database;
 use Sympa::Log;
 
@@ -43,6 +45,13 @@ sub _open {
     return undef unless $db and $db->connect;
     $self->{_db} = $db;
 
+    my $pagesize = $options{pagesize} || $self->{pagesize};
+    if ($pagesize and $db->__dbh->root_dse->supported_control(
+            Net::LDAP::Constant::LDAP_CONTROL_PAGED()
+        )) {
+        $self->{_page} = Net::LDAP::Control::Paged->new( size => $pagesize );
+    }
+
     my $mesg = $self->_open_operation(%options);
     return undef unless $mesg;
 
@@ -61,19 +70,29 @@ sub _open_operation {
     my $ldap_attrs  = $options{attrs}  || $self->{attrs};
     my $ldap_scope  = $options{scope}  || $self->{scope};
 
-    my $mesg = $self->{_db}->do_operation(
-        'search',
+    my @args = (
         base   => $ldap_suffix,
         filter => $ldap_filter,
         attrs  => [split /\s*,\s*/, $ldap_attrs],
-        scope  => $ldap_scope
+        scope  => $ldap_scope,
+        control=> $self->{_page} ? [$self->{_page}] : []
     );
+
+    my $mesg = $self->{_db}->do_operation('search', @args);
+
     unless ($mesg) {
         $log->syslog(
             'err',
             'LDAP search (single level) failed: %s with data source %s',
             $self->{_db}->error, $self
         );
+
+        if ($self->{_page}) {
+            # We had an abnormal exit, so let the server know we do not want any more
+            $self->{_page}->size(0);
+            $self->{_db}->do_operation('search', @args);
+        }
+
         return undef;
     }
 
@@ -108,7 +127,16 @@ sub _load_next {
     }
 
     my @retrieved;
-    my $mesg = $self->__dsh;
+    my $mesg;
+    my $cookie = $self->{_page} ? $self->{_page}->cookie : undef;
+    if (defined($cookie) and length($cookie)) {
+        # second page, or later one (but not post-last) of a paged search:
+        # load next page
+        $mesg = $self->_open_operation(%options);
+    }
+    else {
+        $mesg = $self->__dsh;
+    }
     while (my $entry = $mesg->shift_entry) {
         my $key_values = $entry->get_value($key_attr, asref => 1);
         next unless $key_values and @$key_values;
@@ -151,6 +179,13 @@ sub _load_next {
         }
     }
 
+    if ($self->{_page} and $mesg) {
+        my $cookie = $mesg->control(
+            Net::LDAP::Constant::LDAP_CONTROL_PAGED
+        )->cookie;
+        $self->{_page}->cookie( $cookie );
+    }
+
     return [@retrieved];
 }
 

diff --git a/src/lib/Sympa/DataSource/LDAP2.pm b/src/lib/Sympa/DataSource/LDAP2.pm
@@ -49,16 +49,23 @@ sub _open {
     my @values;
     while (
         my $entry = $self->SUPER::_next(
-            attrs  => $self->{attrs1},
-            select => $self->{select1},
-            regex  => $self->{regex1},
-            turn   => 'first'
+            filter  => $self->{filter1},
+            attrs   => $self->{attrs1},
+            select  => $self->{select1},
+            regex   => $self->{regex1},
+            timeout => $self->{timeout1},
+            suffix  => $self->{suffix1},
+            scope   => $self->{scope1},
+            turn    => 'first'
         )
     ) {
         push @values, $entry->[0] if defined $entry->[0];
     }
     $self->{_attr1values} = [@values];
 
+    # Don't paginate subsequent children searches, they return only one record
+    $self->{_page} = undef;
+
     return 1;
 }