From eb4ca9be355c3abfbb70f0e90b0c6c22cab1e3ca Mon Sep 17 00:00:00 2001
From: Max Horn <max@quendi.de>
Date: Thu, 9 Jan 2025 08:32:37 +0100
Subject: [PATCH] Fix IsomorphismPermGroupOrFailFpGroup(G,max) (#5900)

... to honor its second argument, which limits the coset table size
that gets used before it gives up.
---
 lib/grpfp.gi                                  | 23 +++++++++++++++----
 tst/testbugfix/2024-07-29-fpgroup-enum.tst    | 11 +++++++++
 ...1-09-IsomorphismPermGroupOrFailFpGroup.tst |  6 +++++
 3 files changed, 35 insertions(+), 5 deletions(-)
 create mode 100644 tst/testbugfix/2024-07-29-fpgroup-enum.tst
 create mode 100644 tst/testbugfix/2025-01-09-IsomorphismPermGroupOrFailFpGroup.tst

diff --git a/lib/grpfp.gi b/lib/grpfp.gi
index 8b4013d397..5b86e748e4 100644
--- a/lib/grpfp.gi
+++ b/lib/grpfp.gi
@@ -3899,15 +3899,20 @@ end);
 #M  Size( <G> )  . . . . . . . . . . . . . size of a finitely presented group
 ##
 BindGlobal("SIZE_FP_FROM_CYCLIC_INDEX",
-function( G )
+function( G, max... )  # max = maximal coset table length required
 local   fgens,      # generators of the free group
         rels,       # relators of <G>
         H,          # subgroup of <G>
         gen,        # generator of cyclic subgroup
-        max,        # maximal coset table length required
         e,
         T;          # coset table of <G> by <H>
 
+  if Length(max) = 0 then
+      max := infinity;
+  else
+      max := max[1];
+  fi;
+
   fgens := FreeGeneratorsOfFpGroup( G );
   rels  := RelatorsOfFpGroup( G );
 
@@ -3926,8 +3931,13 @@ local   fgens,      # generators of the free group
     fi;
     # the group could be quite big -- try to find a cyclic subgroup of
     # finite index.
-    gen:=FinIndexCyclicSubgroupGenerator(G,infinity);
-    max:=gen[2];
+    gen:=FinIndexCyclicSubgroupGenerator(G,max);
+    if gen = fail then
+      return fail;
+    fi;
+    if max = infinity then
+      max:=gen[2];
+    fi;
     gen:=gen[1];
 
     H := Subgroup(G,[gen]);
@@ -4049,7 +4059,10 @@ local mappow, G, max, p, gens, rels, comb, i, l, m, H, HH, t, sz,
 
   H:=[]; # indicate pseudo-size 0
   if not HasSize(G) then
-    sz:=SIZE_FP_FROM_CYCLIC_INDEX(G);
+    sz:=SIZE_FP_FROM_CYCLIC_INDEX(G, max);
+    if sz = fail then
+      return fail;
+    fi;
     SetSize(G,sz);
   fi;
   if Size(G)=infinity then
diff --git a/tst/testbugfix/2024-07-29-fpgroup-enum.tst b/tst/testbugfix/2024-07-29-fpgroup-enum.tst
new file mode 100644
index 0000000000..f9627c3df3
--- /dev/null
+++ b/tst/testbugfix/2024-07-29-fpgroup-enum.tst
@@ -0,0 +1,11 @@
+# Sometimes GAP was able to compute the size of an fp group but then
+# for any further action failed to compute a permutation representation.
+# See https://github.com/gap-system/gap/issues/5764 for the report,
+# and https://github.com/gap-system/gap/pull/5770 for the fix.
+gap> f := FreeGroup("a","b","c");;
+gap> g := f / [ f.1*f.1*f.1,f.1*f.2*f.3*f.1*f.3^-1*f.2*f.3*f.3,f.1*f.3*f.2^-1 ];
+<fp group on the generators [ a, b, c ]>
+gap> Size(g);
+84
+gap> IdGroup(g);
+[ 84, 1 ]
diff --git a/tst/testbugfix/2025-01-09-IsomorphismPermGroupOrFailFpGroup.tst b/tst/testbugfix/2025-01-09-IsomorphismPermGroupOrFailFpGroup.tst
new file mode 100644
index 0000000000..808d5dfb63
--- /dev/null
+++ b/tst/testbugfix/2025-01-09-IsomorphismPermGroupOrFailFpGroup.tst
@@ -0,0 +1,6 @@
+# IsomorphismPermGroupOrFailFpGroup ignored its second argument
+# which is supposed to limit the number of cosets that get defined
+# before it gives up
+gap> F:=FreeGroup(2);;G:=F/[F.1^2, F.2^2];;
+gap> IsomorphismPermGroupOrFailFpGroup(G, 100);
+fail