Unicode conversion fix suggested by Jan Varga...

--------------------------------------------------
Subject: Bug in unicode conversion ...
From: Jan Varga <varga@utcru.sk>
To: t-ishii@sra.co.jp
Date: Sat, 18 Nov 2000 17:41:20 +0100 (CET)


Hi,

I tried this new feature in PostgreSQL. I found one bug.
Script UCS_to_8859.pl skips input lines which
1. code <0x80 or
2. ucs <0x100

I think second one is not good idea because some codes in ISO8859-2
have ucs <0x100 (e.g. 0xE9 - 0x00E9)
--------------------------------------------------

Author: tatsuo-ishii

src/backend/utils/mb/Unicode/UCS_to_8859.pl

@@ -2,7 +2,7 @@
 #
 # Copyright 2001 by PostgreSQL Global Development Group
 #
-# $Id: UCS_to_8859.pl,v 1.1 2000/10/30 10:40:29 ishii Exp $
+# $Id: UCS_to_8859.pl,v 1.2 2000/11/26 10:40:43 ishii Exp $
 #
 # Generate UTF-8 <--> ISO8859 code conversion tables from
 # map files provided by Unicode organization.
@@ -36,7 +36,7 @@
 		( $c, $u, $rest ) = split;
 		$ucs = hex($u);
 		$code = hex($c);
-		if( $code >= 0x80 && $ucs >= 0x100 ){
+		if( $code >= 0x80){
 			$utf = &ucs2utf($ucs);
 			if( $array{ $utf } ne "" ){
 				printf STDERR "Warning: duplicate unicode: %04x\n",$ucs;
@@ -80,7 +80,7 @@
 		( $c, $u, $rest ) = split;
 		$ucs = hex($u);
 		$code = hex($c);
-		if( $code >= 0x80 && $ucs >= 0x100 ){
+		if($code >= 0x80){
 			$utf = &ucs2utf($ucs);
 			if( $array{ $utf } ne "" ){
 				printf STDERR "Warning: duplicate unicode: %04x\n",$ucs;

src/backend/utils/mb/Unicode/iso8859_2_to_utf8.map

@@ -1,21 +1,61 @@
-static pg_local_to_utf LUmapISO8859_2[ 57 ] = {
+static pg_local_to_utf LUmapISO8859_2[ 128 ] = {
+  {0x0080, 0xc280},
+  {0x0081, 0xc281},
+  {0x0082, 0xc282},
+  {0x0083, 0xc283},
+  {0x0084, 0xc284},
+  {0x0085, 0xc285},
+  {0x0086, 0xc286},
+  {0x0087, 0xc287},
+  {0x0088, 0xc288},
+  {0x0089, 0xc289},
+  {0x008a, 0xc28a},
+  {0x008b, 0xc28b},
+  {0x008c, 0xc28c},
+  {0x008d, 0xc28d},
+  {0x008e, 0xc28e},
+  {0x008f, 0xc28f},
+  {0x0090, 0xc290},
+  {0x0091, 0xc291},
+  {0x0092, 0xc292},
+  {0x0093, 0xc293},
+  {0x0094, 0xc294},
+  {0x0095, 0xc295},
+  {0x0096, 0xc296},
+  {0x0097, 0xc297},
+  {0x0098, 0xc298},
+  {0x0099, 0xc299},
+  {0x009a, 0xc29a},
+  {0x009b, 0xc29b},
+  {0x009c, 0xc29c},
+  {0x009d, 0xc29d},
+  {0x009e, 0xc29e},
+  {0x009f, 0xc29f},
+  {0x00a0, 0xc2a0},
   {0x00a1, 0xc484},
   {0x00a2, 0xcb98},
   {0x00a3, 0xc581},
+  {0x00a4, 0xc2a4},
   {0x00a5, 0xc4bd},
   {0x00a6, 0xc59a},
+  {0x00a7, 0xc2a7},
+  {0x00a8, 0xc2a8},
   {0x00a9, 0xc5a0},
   {0x00aa, 0xc59e},
   {0x00ab, 0xc5a4},
   {0x00ac, 0xc5b9},
+  {0x00ad, 0xc2ad},
   {0x00ae, 0xc5bd},
   {0x00af, 0xc5bb},
+  {0x00b0, 0xc2b0},
   {0x00b1, 0xc485},
   {0x00b2, 0xcb9b},
   {0x00b3, 0xc582},
+  {0x00b4, 0xc2b4},
   {0x00b5, 0xc4be},
   {0x00b6, 0xc59b},
   {0x00b7, 0xcb87},
+  {0x00b8, 0xc2b8},
   {0x00b9, 0xc5a1},
   {0x00ba, 0xc59f},
   {0x00bb, 0xc5a5},
@@ -24,36 +64,67 @@ static pg_local_to_utf LUmapISO8859_2[ 57 ] = {
   {0x00be, 0xc5be},
   {0x00bf, 0xc5bc},
   {0x00c0, 0xc594},
+  {0x00c1, 0xc381},
+  {0x00c2, 0xc382},
   {0x00c3, 0xc482},
+  {0x00c4, 0xc384},
   {0x00c5, 0xc4b9},
   {0x00c6, 0xc486},
+  {0x00c7, 0xc387},
   {0x00c8, 0xc48c},
+  {0x00c9, 0xc389},
   {0x00ca, 0xc498},
+  {0x00cb, 0xc38b},
   {0x00cc, 0xc49a},
+  {0x00cd, 0xc38d},
+  {0x00ce, 0xc38e},
   {0x00cf, 0xc48e},
   {0x00d0, 0xc490},
   {0x00d1, 0xc583},
   {0x00d2, 0xc587},
+  {0x00d3, 0xc393},
+  {0x00d4, 0xc394},
   {0x00d5, 0xc590},
+  {0x00d6, 0xc396},
+  {0x00d7, 0xc397},
   {0x00d8, 0xc598},
   {0x00d9, 0xc5ae},
+  {0x00da, 0xc39a},
   {0x00db, 0xc5b0},
+  {0x00dc, 0xc39c},
+  {0x00dd, 0xc39d},
   {0x00de, 0xc5a2},
+  {0x00df, 0xc39f},
   {0x00e0, 0xc595},
+  {0x00e1, 0xc3a1},
+  {0x00e2, 0xc3a2},
   {0x00e3, 0xc483},
+  {0x00e4, 0xc3a4},
   {0x00e5, 0xc4ba},
   {0x00e6, 0xc487},
+  {0x00e7, 0xc3a7},
   {0x00e8, 0xc48d},
+  {0x00e9, 0xc3a9},
   {0x00ea, 0xc499},
+  {0x00eb, 0xc3ab},
   {0x00ec, 0xc49b},
+  {0x00ed, 0xc3ad},
+  {0x00ee, 0xc3ae},
   {0x00ef, 0xc48f},
   {0x00f0, 0xc491},
   {0x00f1, 0xc584},
   {0x00f2, 0xc588},
+  {0x00f3, 0xc3b3},
+  {0x00f4, 0xc3b4},
   {0x00f5, 0xc591},
+  {0x00f6, 0xc3b6},
+  {0x00f7, 0xc3b7},
   {0x00f8, 0xc599},
   {0x00f9, 0xc5af},
+  {0x00fa, 0xc3ba},
   {0x00fb, 0xc5b1},
+  {0x00fc, 0xc3bc},
+  {0x00fd, 0xc3bd},
   {0x00fe, 0xc5a3},
   {0x00ff, 0xcb99}
 };

src/backend/utils/mb/Unicode/iso8859_3_to_utf8.map

@@ -1,29 +1,122 @@
-static pg_local_to_utf LUmapISO8859_3[ 28 ] = {
+static pg_local_to_utf LUmapISO8859_3[ 121 ] = {
+  {0x0080, 0xc280},
+  {0x0081, 0xc281},
+  {0x0082, 0xc282},
+  {0x0083, 0xc283},
+  {0x0084, 0xc284},
+  {0x0085, 0xc285},
+  {0x0086, 0xc286},
+  {0x0087, 0xc287},
+  {0x0088, 0xc288},
+  {0x0089, 0xc289},
+  {0x008a, 0xc28a},
+  {0x008b, 0xc28b},
+  {0x008c, 0xc28c},
+  {0x008d, 0xc28d},
+  {0x008e, 0xc28e},
+  {0x008f, 0xc28f},
+  {0x0090, 0xc290},
+  {0x0091, 0xc291},
+  {0x0092, 0xc292},
+  {0x0093, 0xc293},
+  {0x0094, 0xc294},
+  {0x0095, 0xc295},
+  {0x0096, 0xc296},
+  {0x0097, 0xc297},
+  {0x0098, 0xc298},
+  {0x0099, 0xc299},
+  {0x009a, 0xc29a},
+  {0x009b, 0xc29b},
+  {0x009c, 0xc29c},
+  {0x009d, 0xc29d},
+  {0x009e, 0xc29e},
+  {0x009f, 0xc29f},
+  {0x00a0, 0xc2a0},
   {0x00a1, 0xc4a6},
   {0x00a2, 0xcb98},
+  {0x00a3, 0xc2a3},
+  {0x00a4, 0xc2a4},
   {0x00a6, 0xc4a4},
+  {0x00a7, 0xc2a7},
+  {0x00a8, 0xc2a8},
   {0x00a9, 0xc4b0},
   {0x00aa, 0xc59e},
   {0x00ab, 0xc49e},
   {0x00ac, 0xc4b4},
+  {0x00ad, 0xc2ad},
   {0x00af, 0xc5bb},
+  {0x00b0, 0xc2b0},
   {0x00b1, 0xc4a7},
+  {0x00b2, 0xc2b2},
+  {0x00b3, 0xc2b3},
+  {0x00b4, 0xc2b4},
+  {0x00b5, 0xc2b5},
   {0x00b6, 0xc4a5},
+  {0x00b7, 0xc2b7},
+  {0x00b8, 0xc2b8},
   {0x00b9, 0xc4b1},
   {0x00ba, 0xc59f},
   {0x00bb, 0xc49f},
   {0x00bc, 0xc4b5},
+  {0x00bd, 0xc2bd},
   {0x00bf, 0xc5bc},
+  {0x00c0, 0xc380},
+  {0x00c1, 0xc381},
+  {0x00c2, 0xc382},
+  {0x00c4, 0xc384},
   {0x00c5, 0xc48a},
   {0x00c6, 0xc488},
+  {0x00c7, 0xc387},
+  {0x00c8, 0xc388},
+  {0x00c9, 0xc389},
+  {0x00ca, 0xc38a},
+  {0x00cb, 0xc38b},
+  {0x00cc, 0xc38c},
+  {0x00cd, 0xc38d},
+  {0x00ce, 0xc38e},
+  {0x00cf, 0xc38f},
+  {0x00d1, 0xc391},
+  {0x00d2, 0xc392},
+  {0x00d3, 0xc393},
+  {0x00d4, 0xc394},
   {0x00d5, 0xc4a0},
+  {0x00d6, 0xc396},
+  {0x00d7, 0xc397},
   {0x00d8, 0xc49c},
+  {0x00d9, 0xc399},
+  {0x00da, 0xc39a},
+  {0x00db, 0xc39b},
+  {0x00dc, 0xc39c},
   {0x00dd, 0xc5ac},
   {0x00de, 0xc59c},
+  {0x00df, 0xc39f},
+  {0x00e0, 0xc3a0},
+  {0x00e1, 0xc3a1},
+  {0x00e2, 0xc3a2},
+  {0x00e4, 0xc3a4},
   {0x00e5, 0xc48b},
   {0x00e6, 0xc489},
+  {0x00e7, 0xc3a7},
+  {0x00e8, 0xc3a8},
+  {0x00e9, 0xc3a9},
+  {0x00ea, 0xc3aa},
+  {0x00eb, 0xc3ab},
+  {0x00ec, 0xc3ac},
+  {0x00ed, 0xc3ad},
+  {0x00ee, 0xc3ae},
+  {0x00ef, 0xc3af},
+  {0x00f1, 0xc3b1},
+  {0x00f2, 0xc3b2},
+  {0x00f3, 0xc3b3},
+  {0x00f4, 0xc3b4},
   {0x00f5, 0xc4a1},
+  {0x00f6, 0xc3b6},
+  {0x00f7, 0xc3b7},
   {0x00f8, 0xc49d},
+  {0x00f9, 0xc3b9},
+  {0x00fa, 0xc3ba},
+  {0x00fb, 0xc3bb},
+  {0x00fc, 0xc3bc},
   {0x00fd, 0xc5ad},
   {0x00fe, 0xc59d},
   {0x00ff, 0xcb99}