Replace some strtok() with strsep()

strtok() considers adjacent delimiters to be one delimiter, which is
arguably the wrong behavior in some cases.  Replace with strsep(),
which has the right behavior: Adjacent delimiters create an empty
token.

Affected by this are parsing of:

- Stored SCRAM secrets
  ("SCRAM-SHA-256$<iterations>:<salt>$<storedkey>:<serverkey>")

- ICU collation attributes
  ("und@colStrength=primary;colCaseLevel=yes") for ICU older than
  version 54

- PG_COLORS environment variable
  ("error=01;31:warning=01;35:note=01;36:locus=01")

- pg_regress command-line options with comma-separated list arguments
  (--dbname, --create-role) (currently only used pg_regress_ecpg)

Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Reviewed-by: David Steele <david@pgmasters.net>
Discussion: https://www.postgresql.org/message-id/flat/79692bf9-17d3-41e6-b9c9-fc8c3944222a@eisentraut.org

Author: petere

src/backend/libpq/auth-scram.c

@@ -608,16 +608,15 @@ parse_scram_secret(const char *secret, int *iterations,
 	 * SCRAM-SHA-256$<iterations>:<salt>$<storedkey>:<serverkey>
 	 */
 	v = pstrdup(secret);
-	if ((scheme_str = strtok(v, "$")) == NULL)
+	if ((scheme_str = strsep(&v, "$")) == NULL)
 		goto invalid_secret;
-	if ((iterations_str = strtok(NULL, ":")) == NULL)
+	if ((iterations_str = strsep(&v, ":")) == NULL)
 		goto invalid_secret;
-	if ((salt_str = strtok(NULL, "$")) == NULL)
+	if ((salt_str = strsep(&v, "$")) == NULL)
 		goto invalid_secret;
-	if ((storedkey_str = strtok(NULL, ":")) == NULL)
-		goto invalid_secret;
-	if ((serverkey_str = strtok(NULL, "")) == NULL)
+	if ((storedkey_str = strsep(&v, ":")) == NULL)
 		goto invalid_secret;
+	serverkey_str = v;
 
 	/* Parse the fields */
 	if (strcmp(scheme_str, "SCRAM-SHA-256") != 0)

src/backend/utils/adt/pg_locale.c

@@ -2813,6 +2813,7 @@ icu_set_collation_attributes(UCollator *collator, const char *loc,
 	char	   *icu_locale_id;
 	char	   *lower_str;
 	char	   *str;
+	char	   *token;
 
 	/*
 	 * The input locale may be a BCP 47 language tag, e.g.
@@ -2838,7 +2839,7 @@ icu_set_collation_attributes(UCollator *collator, const char *loc,
 		return;
 	str++;
 
-	for (char *token = strtok(str, ";"); token; token = strtok(NULL, ";"))
+	while ((token = strsep(&str, ";")))
 	{
 		char	   *e = strchr(token, '=');
 

src/common/logging.c

@@ -119,7 +119,9 @@ pg_logging_init(const char *argv0)
 
 			if (colors)
 			{
-				for (char *token = strtok(colors, ":"); token; token = strtok(NULL, ":"))
+				char	   *token;
+
+				while ((token = strsep(&colors, ":")))
 				{
 					char	   *e = strchr(token, '=');
 

src/test/regress/pg_regress.c

@@ -234,12 +234,11 @@ static void
 split_to_stringlist(const char *s, const char *delim, _stringlist **listhead)
 {
 	char	   *sc = pg_strdup(s);
-	char	   *token = strtok(sc, delim);
+	char	   *token;
 
-	while (token)
+	while ((token = strsep(&sc, delim)))
 	{
 		add_stringlist_item(listhead, token);
-		token = strtok(NULL, delim);
 	}
 	free(sc);
 }