Fix IsomorphismPermGroupOrFailFpGroup to honor its second argument, which limits the coset table size that gets used before it gives up

lib/grpfp.gi

@@ -3899,15 +3899,20 @@
 #M  Size( <G> )  . . . . . . . . . . . . . size of a finitely presented group
 ##
 BindGlobal("SIZE_FP_FROM_CYCLIC_INDEX",
-function( G )
+function( G, max... )  # max = maximal coset table length required
 local   fgens,      # generators of the free group
         rels,       # relators of <G>
         H,          # subgroup of <G>
         gen,        # generator of cyclic subgroup
-        max,        # maximal coset table length required
         e,
         T;          # coset table of <G> by <H>
 
+  if Length(max) = 0 then
+      max := infinity;
+  else
+      max := max[1];
+  fi;
+
   fgens := FreeGeneratorsOfFpGroup( G );
   rels  := RelatorsOfFpGroup( G );
 
@@ -3926,8 +3931,13 @@
     fi;
     # the group could be quite big -- try to find a cyclic subgroup of
     # finite index.
-    gen:=FinIndexCyclicSubgroupGenerator(G,infinity);
-    max:=gen[2];
+    gen:=FinIndexCyclicSubgroupGenerator(G,max);
+    if gen = fail then
+      return fail;
+    fi;
+    if max = infinity then
+      max:=gen[2];
+    fi;
     gen:=gen[1];
 
     H := Subgroup(G,[gen]);
@@ -4049,7 +4059,10 @@
 
   H:=[]; # indicate pseudo-size 0
   if not HasSize(G) then
-    sz:=SIZE_FP_FROM_CYCLIC_INDEX(G);
+    sz:=SIZE_FP_FROM_CYCLIC_INDEX(G, max);
+    if sz = fail then
+      return fail;
+    fi;
     SetSize(G,sz);
   fi;
   if Size(G)=infinity then

tst/testbugfix/2024-07-29-fpgroup-enum.tst

@@ -0,0 +1,11 @@
+# Sometimes GAP was able to compute the size of an fp group but then
+# for any further action failed to compute a permutation representation.
+# See https://github.com/gap-system/gap/issues/5764 for the report,
+# and https://github.com/gap-system/gap/pull/5770 for the fix.
+gap> f := FreeGroup("a","b","c");;
+gap> g := f / [ f.1*f.1*f.1,f.1*f.2*f.3*f.1*f.3^-1*f.2*f.3*f.3,f.1*f.3*f.2^-1 ];
+<fp group on the generators [ a, b, c ]>
+gap> Size(g);
+84
+gap> IdGroup(g);
+[ 84, 1 ]

tst/testbugfix/2025-01-09-IsomorphismPermGroupOrFailFpGroup.tst

@@ -0,0 +1,6 @@
+# IsomorphismPermGroupOrFailFpGroup ignored its second argument
+# which is supposed to limit the number of cosets that get defined
+# before it gives up
+gap> F:=FreeGroup(2);;G:=F/[F.1^2, F.2^2];;
+gap> IsomorphismPermGroupOrFailFpGroup(G, 100);
+fail