HTTP Body handling different when redirecting?

[Seanstoppable]

I noticed this when reviewing the http redirect bug I introduced.

We have this block of code in getCheckRedirect:

zgrab2/modules/http/scanner.go

Lines 322 to 341 in b9e2158

	b := new(bytes.Buffer)
	maxReadLen := int64(scan.scanner.config.MaxSize) * 1024
	readLen := maxReadLen
	if res.ContentLength >= 0 && res.ContentLength < maxReadLen {
	readLen = res.ContentLength
	}
	bytesRead, _ := io.CopyN(b, res.Body, readLen)
	if scan.scanner.config.WithBodyLength {
	res.BodyTextLength = bytesRead
	}
	res.BodyText = b.String()
	if len(res.BodyText) > 0 {
	if scan.scanner.decodedHashFn != nil {
	res.BodyHash = scan.scanner.decodedHashFn([]byte(res.BodyText))
	} else {
	m := sha256.New()
	m.Write(b.Bytes())
	res.BodySHA256 = m.Sum(nil)
	}
	}

and this much LARGER block in the overall Grab method:

zgrab2/modules/http/scanner.go

Lines 494 to 575 in b9e2158

	buf := new(bytes.Buffer)
	maxReadLen := int64(scan.scanner.config.MaxSize) * 1024
	readLen := maxReadLen
	if resp.ContentLength >= 0 && resp.ContentLength < maxReadLen {
	readLen = resp.ContentLength
	}
	if n, err := io.CopyN(buf, resp.Body, readLen); err != nil && !strings.Contains(err.Error(), "EOF") {
	return zgrab2.NewScanError(zgrab2.SCAN_UNKNOWN_ERROR, fmt.Errorf("error populating response body after %d bytes: %w", n, err))
	}
	encoder, encoding, certain := charset.DetermineEncoding(buf.Bytes(), resp.Header.Get("content-type"))
	bodyText := ""
	decodedSuccessfully := false
	decoder := encoder.NewDecoder()
	//"windows-1252" is the default value and will likely not decode correctly
	if certain || encoding != "windows-1252" {
	decoded, decErr := decoder.Bytes(buf.Bytes())
	if decErr == nil {
	bodyText = string(decoded)
	decodedSuccessfully = true
	}
	}
	if !decodedSuccessfully {
	bodyText = buf.String()
	}
	// Application-specific logic for retrying HTTP as HTTPS; if condition matches, return protocol error
	bodyTextLen := int64(len(bodyText))
	if scan.scanner.config.FailHTTPToHTTPS && scan.results.Response.StatusCode == 400 && bodyTextLen < 1024 && bodyTextLen > 24 {
	// Apache: "You're speaking plain HTTP to an SSL-enabled server port"
	// NGINX: "The plain HTTP request was sent to HTTPS port"
	var sliceLen int64 = 128
	if readLen < sliceLen {
	sliceLen = readLen
	}
	if bodyTextLen < sliceLen {
	sliceLen = bodyTextLen
	}
	sliceBuf := bodyText[:sliceLen]
	if strings.Contains(sliceBuf, "The plain HTTP request was sent to HTTPS port") ||
	strings.Contains(sliceBuf, "You're speaking plain HTTP") ||
	strings.Contains(sliceBuf, "combination of host and port requires TLS") ||
	strings.Contains(sliceBuf, "Client sent an HTTP request to an HTTPS server") {
	return zgrab2.NewScanError(zgrab2.SCAN_PROTOCOL_ERROR, errors.New("NGINX or Apache HTTP over HTTPS failure"))
	}
	}
	// re-enforce readlen
	if int64(len(bodyText)) > readLen {
	scan.results.Response.BodyText = bodyText[:int(readLen)]
	} else {
	scan.results.Response.BodyText = bodyText
	}
	if scan.scanner.config.WithBodyLength {
	scan.results.Response.BodyTextLength = int64(len(scan.results.Response.BodyText))
	}
	if len(scan.results.Response.BodyText) > 0 {
	if scan.scanner.decodedHashFn != nil {
	scan.results.Response.BodyHash = scan.scanner.decodedHashFn([]byte(scan.results.Response.BodyText))
	} else {
	m := sha256.New()
	m.Write(buf.Bytes())
	scan.results.Response.BodySHA256 = m.Sum(nil)
	}
	}
	// Check if the BodyText is binary, we'll need to base64 encode it
	// This occurs after length enforcement, since readLen is the size of data read on the wire, not encoded
	if !utf8.ValidString(scan.results.Response.BodyText) {
	// body isn't valid UTF-8, so we need to base64 encode it
	// without this, binary data gets set as
	scan.results.Response.BodyText = base64.StdEncoding.EncodeToString([]byte(scan.results.Response.BodyText))
	}
	return nil

These seem to be trying to do the same kind of parsing, but the logic of the two diverges in some respects.