Optionally reallocate large+small buffer after each module

check/features.frm

@@ -1111,3 +1111,27 @@ format C;
 assert succeeded?
 assert !(file("out.c") =~ /[_] [+]=  /)
 *--#] Issue392_ContinuationLines_0 :
+*--#[ Sortrealloc_1 :
+On sortreallocate;
+Symbol x,y;
+Local F = (x+y)^10;
+.sort
+Identify x = - y;
+.sort
+Print +s;
+.end
+assert succeeded?
+assert result("F") =~ expr("0");
+*--#] Sortrealloc_1 :
+*--#[ Sortrealloc_2 :
+Symbol x,y;
+Local F = (x+y)^10;
+.sort
+#sortreallocate
+Identify x = - y;
+.sort
+Print +s;
+.end
+assert succeeded?
+assert result("F") =~ expr("0");
+*--#] Sortrealloc_2 :

doc/manual/prepro.tex

@@ -1936,6 +1936,22 @@ \section{\#skipextrasymbols}
 incorrect.
 
 %--#] skipextrasymbols : 
+%--#[ sortreallocate :
+
+\section{\#sortreallocate}
+\label{presortreallocate}
+
+\noindent Syntax:
+
+\#sortreallocate\index{\#sortreallocate}
+
+\noindent See also ``On sortreallocate;'' (\ref{substaon}).
+
+\noindent Reallocates the small and large buffer (also on the worker threads)
+at the end of the current module. In some cases this can significantly reduce
+\FORM's memory usage as measured by ``resident set size''.
+
+%--#] sortreallocate : 
 %--#[ switch :
 
 \section{\#switch}

doc/manual/setup.tex

@@ -439,6 +439,14 @@ \chapter{The setup}
 with 1/N times the size of the buffer of the master. This may get made a 
 bit bigger when potential conflicts with MaxTermSize occur.
 
+The (typically) largest buffers (the small and large buffers) may be reallocated
+at the end of a single module (see \#sortreallocate (\ref{presortreallocate})) or
+at the end of each module (see ``On sortreallocate;'' (\ref{substaon})). In some
+cases this can significantly reduce \FORM's memory usage as measured by
+``resident set size''. For programs which consist of a large number of very
+quickly-running modules, this can incur a noticable performance penalty if performed
+every module.
+
 The default settings are
 \begin{center}
 \begin{tabular}{lrrr}

doc/manual/statements.tex

@@ -3836,6 +3836,10 @@ \section{off}
 \rightvitem{13cm}{Takes the writing of the statistics back from shorthand 
 mode to the regular statistics mode in which each statistics messages takes 
 three lines of text and one blank line.}
+
+\leftvitem{3.5cm}{sortreallocate\index{off!sortreallocate}}
+\rightvitem{13cm}{Turns off the reallocation of the small and large buffer
+at the end of each module.}
 
 \leftvitem{3.5cm}{threadloadbalancing\index{off!threadloadbalancing}}
 \rightvitem{13cm}{\vspace{1.5ex}Disables the loadbalancing mechanism of 
@@ -4001,6 +4005,14 @@ \section{on}
 \leftvitem{3.5cm}{shortstats\index{on!shortstats}}
 \rightvitem{13cm}{Same as `On shortstatistics'.}
 
+\leftvitem{3.5cm}{sortreallocate\index{on!sortreallocate}}
+\rightvitem{13cm}{Reallocate the small and large buffer (also on the worker
+threads) at the end of every module. In some cases this can significantly reduce
+\FORM's memory usage as measured by ``resident set size''. For programs which
+consist of a large number of very quickly-running modules, this can incur a
+noticable performance penalty. See also \#sortreallocate (\ref{presortreallocate})
+for a single-module version of this feature.}
+
 \leftvitem{3.5cm}{statistics\index{on!statistics}}
 \rightvitem{13cm}{Turns the writing of runtime statistics on. This is the 
 default. It is possible to change this default with one of the setup 

sources/compcomm.c

@@ -136,6 +136,7 @@ static KEYWORDV onoffoptions[] = {
 	,{"oldgcd", 		&(AC.OldGCDflag),	1,	0}
 	,{"innertest",      &(AC.InnerTest),  1,  0}
 	,{"wtimestats",     &(AC.WTimeStatsFlag),  1,  0}
+	,{"sortreallocate",	&(AC.SortReallocateFlag), 1, 0}
 };
 
 static WORD one = 1;
@@ -627,6 +628,14 @@ int CoOff(UBYTE *s)
 				AC.TestValue = 0;
 			}
 		}
+		else if ( StrICont(t,(UBYTE *)"sortreallocate") == 0 ) {
+			if ( AC.SortReallocateFlag == 2 ) {
+				/* The flag has been set by #sortreallocate, and it was off before. Leave it as 2,
+				   so that the reallocation still happens in the current module. It will be turned
+				   off after the reallocation is done. */
+				return(0);
+			}
+		}
 		*s = c;
 	 	*onoffoptions[i].var = onoffoptions[i].flags; 
 		AR.SortType = AC.SortType;

sources/declare.h

@@ -957,6 +957,7 @@ extern int    DoPrePrintTimes(UBYTE *);
 extern int    DoPreWrite(UBYTE *);
 extern int    DoPreClose(UBYTE *);
 extern int    DoPreRemove(UBYTE *);
+extern int    DoPreSortReallocate(UBYTE *);
 extern int    DoCommentChar(UBYTE *);
 extern int    DoPrcExtension(UBYTE *);
 extern int    DoPreReset(UBYTE *);
@@ -1451,6 +1452,7 @@ extern int    MasterWaitThread(int);
 extern void   WakeupMasterFromThread(int,int);
 extern int    LoadReadjusted(VOID);
 extern int    IniSortBlocks(int);
+extern int    UpdateSortBlocks(int);
 extern int    TreatIndexEntry(PHEAD LONG);
 extern WORD   GetTerm2(PHEAD WORD *);
 extern void	SetHideFiles(VOID);

sources/execute.c

@@ -1046,6 +1046,44 @@ if ( AC.SwitchInArray > 0 ) {
 	  AC.MultiBracketBuf = 0;
 	}
 
+	if ( AC.SortReallocateFlag ) {
+		/* Reallocate the sort buffers to reduce resident set usage */
+		/* AT.SS is the same as AT.S0 here */
+		SORTING* S = AT.S0;
+		M_free(S->lBuffer, "SortReallocate lBuffer+sBuffer");
+		S->lBuffer = Malloc1(sizeof(*(S->lBuffer))*(S->LargeSize+S->SmallEsize), "SortReallocate lBuffer+sBuffer");
+		S->lTop = S->lBuffer+S->LargeSize;
+		S->sBuffer = S->lTop;
+		if ( S->LargeSize == 0 ) { S->lBuffer = 0; S->lTop = 0; }
+		S->sTop = S->sBuffer + S->SmallSize;
+		S->sTop2 = S->sBuffer + S->SmallEsize;
+		S->sHalf = S->sBuffer + (LONG)((S->SmallSize+S->SmallEsize)>>1);
+
+#ifdef WITHPTHREADS
+		/* We have to re-set the pointers into master lBuffer in the SortBlocks */
+		UpdateSortBlocks(AM.totalnumberofthreads-1);
+
+		/* The SortBots do not have a real sort buffer to reallocate. */
+		/* AB[0] has been reallocated above already. */
+		for ( i = 1; i < AM.totalnumberofthreads; i++ ) {
+			SORTING* S = AB[i]->T.S0;
+			M_free(S->lBuffer, "SortReallocate lBuffer+sBuffer");
+			S->lBuffer = Malloc1(sizeof(*(S->lBuffer))*(S->LargeSize+S->SmallEsize), "SortReallocate lBuffer+sBuffer");
+			S->lTop = S->lBuffer+S->LargeSize;
+			S->sBuffer = S->lTop;
+			if ( S->LargeSize == 0 ) { S->lBuffer = 0; S->lTop = 0; }
+			S->sTop = S->sBuffer + S->SmallSize;
+			S->sTop2 = S->sBuffer + S->SmallEsize;
+			S->sHalf = S->sBuffer + (LONG)((S->SmallSize+S->SmallEsize)>>1);
+		}
+#endif
+	}
+	if ( AC.SortReallocateFlag == 2 ) {
+		/* The Flag was set for a single module by the preprocessor #sortreallocate,
+		   so turn it off again. */
+		AC.SortReallocateFlag = 0;
+	}
+
 	return(RetCode);
 }
 

sources/pre.c

@@ -104,6 +104,7 @@ static KEYWORD precommands[] = {
 	,{"setrandom"    , DoSetRandom    , 0, 0}
 	,{"show"         , DoPreShow      , 0, 0}
 	,{"skipextrasymbols" , DoSkipExtraSymbols , 0, 0}
+	,{"sortreallocate", DoPreSortReallocate , 0, 0}
 #ifdef WITHFLOAT
     ,{"startfloat"   , DoStartFloat   , 0, 0}
 #endif
@@ -3777,6 +3778,23 @@ int DoPrePrintTimes(UBYTE *s)
 
 /*
  		#] DoPrePrintTimes : 
+		#[ DoPreSortReallocate :
+*/
+
+int DoPreSortReallocate(UBYTE *s)
+{
+	DUMMYUSE(s);
+	if ( AC.SortReallocateFlag == 0 ) {
+		/* Currently off, so set to 2. Then the reallocation code knows the flag was
+		   set here, since "On sortreallocate;" sets it to 1. */
+		AC.SortReallocateFlag = 2;
+	}
+	/* If the flag is already on, do nothing. */
+	return(0);
+}
+
+/*
+		#] DoPreSortReallocate :
  		#[ DoPreAppend :
 
 		Syntax:

sources/structs.h

@@ -1867,6 +1867,10 @@ struct C_const {
     int     MemDebugFlag;          /* Only used when MALLOCDEBUG in tools.c */
     int     OldGCDflag;
     int     WTimeStatsFlag;
+    int     SortReallocateFlag;    /* Controls reallocation of large+small buffer at module end.
+                                        0 : Off
+                                        1 : On, every module (set by On sortreallocate;)
+                                        2 : On, single module (set by #sortreallocate) */
 	int     doloopstacksize;
 	int     dolooplevel;
     int     CheckpointFlag;        /**< Tells preprocessor whether checkpoint code must executed.

sources/threads.c

@@ -4769,6 +4769,75 @@ int IniSortBlocks(int numworkers)
 
 /*
   	#] IniSortBlocks : 
+  	#[ UpdateSortBlocks :
+*/
+
+/**
+ *	A version of IniSortBlocks which only updates the pointers in the master's
+ *	buffer, to be used after reallocation of that buffer.
+ */
+int UpdateSortBlocks(int numworkers)
+{
+	ALLPRIVATES *B;
+	SORTING *S;
+	LONG totalsize, workersize, blocksize, numberofterms;
+	int maxter, id, j;
+	int numberofblocks = NUMBEROFBLOCKSINSORT, numparts;
+	WORD *w;
+
+	if ( numworkers == 0 ) return(0);
+
+#ifdef WITHSORTBOTS
+	if ( numworkers > 2 ) {
+		numparts = 2*numworkers - 2;
+		numberofblocks = numberofblocks/2;
+	}
+	else {
+		numparts = numworkers;
+	}
+#else
+	numparts = numworkers;
+#endif
+	S = AM.S0;
+	totalsize = S->LargeSize + S->SmallEsize;
+	workersize = totalsize / numparts;
+	maxter = AM.MaxTer/sizeof(WORD);
+	blocksize = ( workersize - maxter )/numberofblocks;
+	numberofterms = blocksize / maxter;
+	if ( numberofterms < MINIMUMNUMBEROFTERMS ) {
+/*
+		This should have been taken care of in RecalcSetups.
+*/
+		MesPrint("We have a problem with the size of the blocks in UpdateSortBlocks");
+		Terminate(-1);
+	}
+/*
+	Layout:  For each worker
+				block 0: size is maxter WORDS
+				numberofblocks blocks of size blocksize WORDS
+*/
+	w = S->lBuffer;
+	if ( w == 0 ) w = S->sBuffer;
+	for ( id = 1; id <= numparts; id++ ) {
+		B = AB[id];
+		AT.SB.MasterFill[0] = AT.SB.MasterStart[0] = w;
+		w += maxter;
+		AT.SB.MasterStop[0] = w;
+		for ( j = 1; j <= numberofblocks; j++ ) {
+			AT.SB.MasterFill[j] = AT.SB.MasterStart[j] = w;
+			w += blocksize;
+			AT.SB.MasterStop[j] = w;
+		}
+	}
+	if ( w > S->sTop2 ) {
+		MesPrint("Counting problem in UpdateSortBlocks");
+		Terminate(-1);
+	}
+	return(0);
+}
+
+/*
+  	#] UpdateSortBlocks : 
   	#[ DefineSortBotTree :
 */