diff --git a/.gitignore b/.gitignore index 2794f157..d7f0fcba 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ wiki *sublime-* coverage /.idea -package-lock.json +/package-lock.json +/.npmrc +/.vscode/settings.json diff --git a/.npmignore b/.npmignore index e1bafe0c..d345f149 100644 --- a/.npmignore +++ b/.npmignore @@ -5,3 +5,4 @@ test wiki coverage .travis.yml +/.npmrc diff --git a/README.md b/README.md index f0137950..0bfaf343 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ * In-browser usage via [Browserify](https://github.com/substack/node-browserify) (~180k gzip compressed with Buffer shim included). * Typescript [type definition file](https://github.com/ashtuchkin/iconv-lite/blob/master/lib/index.d.ts) included. * React Native is supported (need to explicitly `npm install` two more modules: `buffer` and `stream`). + * Transliteration option is available when either [unidecode-plus](https://www.npmjs.com/package/unidecode-plus) or [unidecode](https://www.npmjs.com/package/unidecode) are added to your project * License: MIT. [![NPM Stats](https://nodei.co/npm/iconv-lite.png)](https://npmjs.org/package/iconv-lite/) @@ -30,7 +31,10 @@ str = iconv.decode(Buffer.from([0x68, 0x65, 0x6c, 0x6c, 0x6f]), 'win1251'); buf = iconv.encode("Sample input string", 'win1251'); // Check if encoding is supported -iconv.encodingExists("us-ascii") +iconv.encodingExists("us-ascii"); + +// Convert from js string to an encoded buffer, keeping accented characters like "é", but transliterating Chinese. +buf2 = iconv.encode("Café 北京", 'iso-8859-1', { transliterate: true }); ``` ### Streaming API (Node v0.10+) @@ -143,11 +147,38 @@ This library supports UTF-32LE, UTF-32BE and UTF-32 encodings. Like the UTF-16 e * The default of UTF-32LE can be overridden with the `defaultEncoding: 'utf-32be'` option. Strips BOM unless `stripBOM: false`. * Encoding: uses UTF-32LE and writes BOM by default. Use `addBOM: false` to override. (`defaultEncoding: 'utf-32be'` can also be used here to change encoding.) +## Transliteration + +If either [unidecode-plus](https://www.npmjs.com/package/unidecode-plus) or [unidecode](https://www.npmjs.com/package/unidecode) are added to your project ("`npm install unidecode-plus`" or "`npm install unidecode`"), the option will be available to transliterate characters which are not available in a particular encoding. The transliterations are always plain ASCII characters, but unlike using unidecode directly (which will convert *all* non-ASCII characters into transliterations), transliterations done using iconv will only transliterate characters which are not available in the target character encoding. + +In this example: +``` +buf = iconv.encode("Café 北京", 'iso-8859-1', { transliterate: true }); +``` +The output is ``. Converted back into ISO-8859-1 text, this is "Café Bei Jing ", preserving the accented "é", and only transliterating the Chinese characters. + +Transliteration to a string instead of a buffer can also be done directly, like this: +``` +str = iconv.transliterate("Café 北京", 'iso-8859-1'); +``` +When encoding to create a buffer, the node-iconv style of requesting transliteration can also be used: +``` +buf = iconv.encode("Café 北京", 'iso-8859-1//translit'); +``` + +If you use `unidecode-plus` instead of `unidecode`, you get two additional transliteration options: `german`, and `smartSpacing`. + +The `german` option transliterates `Ä`, `ä`, `Ö`, `ö`, `Ü`, and `ü` to `AE`, `ae`, `OE`, `oe`, `UE`, and `ue`, respectively, instead of just removing the umlauts. + +The `smartSpacing` options improves the formatting of transliterated text, removing some unnecessary spaces, and adding others for clarity. For example, "Café 北京, 鞋 size 10½" becomes "Cafe Bei Jing, Xie size 10 1/2" using `smartSpacing`. Without it, you get "Cafe Bei Jing , Xie size 101/2". (See the [unidecode-plus site](https://github.com/kshetline/unidecode-plus/blob/master/README.md) for more detail.) + +Please take note that transliteration only affects encoding, not decoding. + ## Other notes -When decoding, be sure to supply a Buffer to decode() method, otherwise [bad things usually happen](https://github.com/ashtuchkin/iconv-lite/wiki/Use-Buffers-when-decoding). -Untranslatable characters are set to � or ?. No transliteration is currently supported. -Node versions 0.10.31 and 0.11.13 are buggy, don't use them (see #65, #77). +* When decoding, be sure to supply a Buffer to decode() method, otherwise [bad things usually happen](https://github.com/ashtuchkin/iconv-lite/wiki/Use-Buffers-when-decoding). +* Untranslatable characters are set to � or ? unless using transliteration. +* Node versions 0.10.31 and 0.11.13 are buggy, don't use them (see #65, #77). ## Testing diff --git a/encodings/tables/transliteration-ranges.json b/encodings/tables/transliteration-ranges.json new file mode 100644 index 00000000..072707ca --- /dev/null +++ b/encodings/tables/transliteration-ranges.json @@ -0,0 +1,640 @@ +{"armscii8":[[0,160],[171,171],[187,187],[1329,1366],[1370,1374],[1377,1415],[1417, +1418],[8212,8212],[8230,8230]],"ascii":[[0,127]],"base64":[[0,11]],"big5hkscs":[[0, +127],[167,168],[175,177],[183,183],[192,193],[200,202],[210,211],[215,215],[224,225], +[232,234],[236,237],[242,243],[247,250],[252,252],[256,257],[274,275],[282,283],[299, +299],[331,333],[339,339],[363,363],[461,462],[464,466],[468,468],[470,470],[472,472], +[474,474],[476,476],[592,593],[596,596],[603,603],[609,609],[618,618],[629,629],[643, +643],[650,650],[710,711],[713,715],[717,717],[729,729],[913,929],[931,937],[945,961], +[963,969],[1025,1025],[1040,1103],[1105,1105],[7870,7873],[8211,8212],[8216,8217], +[8220,8221],[8229,8231],[8242,8242],[8245,8245],[8251,8251],[8364,8364],[8451,8451], +[8453,8453],[8457,8457],[8470,8470],[8481,8481],[8544,8553],[8560,8569],[8592,8595], +[8598,8601],[8632,8633],[8679,8679],[8725,8725],[8730,8730],[8734,8736],[8739,8739], +[8741,8741],[8745,8747],[8750,8750],[8756,8757],[8786,8786],[8800,8801],[8806,8807], +[8853,8853],[8857,8857],[8869,8869],[8895,8895],[9178,9179],[9216,9247],[9249,9249], +[9312,9321],[9332,9341],[9472,9472],[9474,9474],[9484,9484],[9488,9488],[9492,9492], +[9496,9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532,9532],[9552,9588], +[9601,9615],[9620,9621],[9632,9633],[9650,9651],[9660,9661],[9670,9671],[9675,9675], +[9678,9679],[9698,9701],[9733,9734],[9792,9792],[9794,9794],[10045,10045],[11904, +11904],[11908,11908],[11910,11912],[11914,11914],[11916,11917],[11925,11925],[11932, +11933],[11941,11941],[11943,11943],[11946,11946],[11948,11948],[11950,11950],[11958, +11958],[11964,11964],[11966,11966],[11974,11974],[11978,11978],[11980,11981],[11983, +11983],[11990,11991],[11998,11998],[12003,12003],[12083,12083],[12288,12291],[12293, +12306],[12308,12309],[12317,12318],[12321,12329],[12353,12435],[12443,12446],[12449, +12534],[12540,12542],[12549,12585],[12736,12751],[12849,12849],[12963,12963],[13198, +13199],[13212,13214],[13217,13217],[13252,13252],[13262,13262],[13265,13266],[13269, +13269],[13365,13365],[13376,13376],[13386,13386],[13388,13388],[13412,13412],[13427, +13427],[13434,13434],[13437,13438],[13459,13459],[13462,13462],[13477,13477],[13487, +13487],[13500,13500],[13505,13505],[13512,13512],[13535,13535],[13540,13540],[13542, +13542],[13563,13563],[13574,13574],[13630,13630],[13649,13649],[13651,13651],[13657, +13657],[13665,13665],[13677,13677],[13680,13680],[13682,13682],[13687,13688],[13700, +13700],[13719,13720],[13729,13729],[13733,13733],[13741,13741],[13759,13759],[13761, +13761],[13765,13765],[13767,13767],[13770,13770],[13774,13774],[13778,13778],[13782, +13782],[13787,13787],[13789,13789],[13809,13811],[13819,13819],[13822,13822],[13833, +13833],[13848,13848],[13850,13850],[13859,13859],[13861,13861],[13869,13869],[13877, +13877],[13881,13881],[13886,13886],[13895,13897],[13902,13902],[13919,13919],[13921, +13921],[13946,13946],[13953,13953],[13978,13978],[13989,13989],[13994,13994],[13996, +13996],[14000,14001],[14005,14005],[14009,14009],[14012,14012],[14017,14017],[14019, +14021],[14023,14024],[14035,14036],[14038,14038],[14045,14045],[14049,14050],[14053, +14054],[14069,14069],[14081,14081],[14083,14083],[14088,14088],[14090,14090],[14093, +14093],[14108,14108],[14114,14115],[14117,14117],[14124,14125],[14128,14128],[14130, +14131],[14138,14138],[14144,14144],[14147,14147],[14178,14178],[14191,14191],[14231, +14231],[14240,14240],[14265,14265],[14270,14270],[14294,14294],[14322,14322]],"binary":[[0, +255],[257,511],[513,767],[769,1023],[1025,1279],[1281,1535],[1537,1791],[1793,2047], +[2049,2303],[2305,2559],[2561,2815],[2817,3071],[3073,3327],[3329,3583],[3585,3839], +[3841,4095],[4097,4351],[4353,4607],[4609,4863],[4865,5119],[5121,5375],[5377,5631], +[5633,5887],[5889,6143],[6145,6399],[6401,6655],[6657,6911],[6913,7167],[7169,7423], +[7425,7679],[7681,7935],[7937,8191],[8193,8447],[8449,8703],[8705,8959],[8961,9215], +[9217,9471],[9473,9727],[9729,9983],[9985,10239],[10241,10495],[10497,10751],[10753, +11007],[11009,11263],[11265,11519],[11521,11775],[11777,12031],[12033,12287],[12289, +12543],[12545,12799],[12801,13055],[13057,13311],[13313,13567],[13569,13823],[13825, +14079],[14081,14335],[14337,14591],[14593,14847],[14849,15103],[15105,15359],[15361, +15615],[15617,15871],[15873,16127],[16129,16383],[16385,16639],[16641,16895],[16897, +17151],[17153,17407],[17409,17663],[17665,17919],[17921,18175],[18177,18431],[18433, +18687],[18689,18943],[18945,19199],[19201,19455],[19457,19711],[19713,19967],[19969, +20223],[20225,20479],[20481,20735],[20737,20991],[20993,21247],[21249,21503],[21505, +21759],[21761,22015],[22017,22271],[22273,22527],[22529,22783],[22785,23039],[23041, +23295],[23297,23551],[23553,23807],[23809,24063],[24065,24319],[24321,24575],[24577, +24831],[24833,25087],[25089,25343],[25345,25599],[25601,25855],[25857,26111],[26113, +26367],[26369,26623],[26625,26879],[26881,27135],[27137,27391],[27393,27647],[27649, +27903],[27905,28159],[28161,28415],[28417,28671],[28673,28927],[28929,29183],[29185, +29439],[29441,29695],[29697,29951],[29953,30207],[30209,30463],[30465,30719],[30721, +30975],[30977,31231],[31233,31487],[31489,31743],[31745,31999],[32001,32255],[32257, +32511],[32513,32767],[32769,33023],[33025,33279],[33281,33535],[33537,33791],[33793, +34047],[34049,34303],[34305,34559],[34561,34815],[34817,35071],[35073,35327],[35329, +35583],[35585,35839],[35841,36095],[36097,36351],[36353,36607],[36609,36863],[36865, +37119],[37121,37375],[37377,37631],[37633,37887],[37889,38143],[38145,38399],[38401, +38655],[38657,38911],[38913,39167],[39169,39423],[39425,39679],[39681,39935],[39937, +40191],[40193,40447],[40449,40703],[40705,40959],[40961,41215],[41217,41471],[41473, +41727],[41729,41983],[41985,42239],[42241,42495],[42497,42751],[42753,43007],[43009, +43263],[43265,43519],[43521,43775],[43777,44031],[44033,44287],[44289,44543],[44545, +44799],[44801,45055],[45057,45311],[45313,45567],[45569,45823],[45825,46079],[46081, +46335],[46337,46591],[46593,46847],[46849,47103],[47105,47359],[47361,47615],[47617, +47871],[47873,48127],[48129,48383],[48385,48639],[48641,48895],[48897,49151],[49153, +49407],[49409,49663],[49665,49919],[49921,50175],[50177,50431],[50433,50687],[50689, +50943],[50945,51199],[51201,51455],[51457,51711],[51713,51967],[51969,52223],[52225, +52479],[52481,52735],[52737,52991],[52993,53247],[53249,53503],[53505,53759],[53761, +54015],[54017,54271],[54273,54527],[54529,54783],[54785,55039],[55041,55295],[55297, +55551],[55553,55807],[55809,56063],[56065,56319],[56321,56575],[56577,56831],[56833, +57087],[57089,57343],[57345,57599],[57601,57855],[57857,58111],[58113,58367],[58369, +58623],[58625,58879],[58881,59135],[59137,59391],[59393,59647],[59649,59903],[59905, +60159],[60161,60415],[60417,60671],[60673,60927],[60929,61183],[61185,61439],[61441, +61695],[61697,61951],[61953,62207],[62209,62463],[62465,62719],[62721,62975],[62977, +63231],[63233,63487],[63491,63491],[63493,63493],[63495,63495],[63497,63497],[63499, +63499],[63501,63501],[63503,63503],[63505,63505]],"cesu8":[[0,2160639]],"cp1046":[[0, +127],[136,136],[160,160],[164,164],[173,173],[215,215],[247,247],[1548,1548],[1563, +1563],[1567,1567],[1569,1591],[1593,1594],[1600,1618],[1632,1641],[9472,9472],[9474, +9474],[9484,9484],[9488,9488],[9492,9492],[9496,9496],[9632,9632],[61684,61692],[63089, +63089],[63095,63095],[63097,63097],[63099,63099],[63101,63101],[63103,63103],[63106, +63106],[63108,63108],[63112,63112],[63114,63115],[63118,63118],[63121,63121],[63127, +63127],[63131,63131],[63135,63135],[63139,63139],[63143,63143],[63155,63155],[63159, +63159],[63163,63163],[63167,63167],[63175,63175],[63178,63180],[63182,63184],[63187, +63187],[63191,63191],[63195,63195],[63199,63199],[63203,63203],[63207,63207],[63209, +63209],[63212,63212],[63216,63216],[63218,63219],[63221,63228]],"cp1124":[[0,160], +[167,167],[173,173],[1025,1026],[1028,1036],[1038,1103],[1105,1106],[1108,1116],[1118, +1119],[1168,1169],[8470,8470]],"cp1125":[[0,127],[160,160],[164,164],[183,183],[1025, +1025],[1028,1028],[1030,1031],[1040,1103],[1105,1105],[1108,1108],[1110,1111],[1168, +1169],[8470,8470],[8730,8730],[9472,9472],[9474,9474],[9484,9484],[9488,9488],[9492, +9492],[9496,9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532,9532],[9552, +9580],[9600,9600],[9604,9604],[9608,9608],[9612,9612],[9616,9619],[9632,9632]],"cp1129":[[0, +167],[169,179],[181,183],[185,194],[196,203],[205,207],[209,209],[211,212],[214,220], +[223,226],[228,235],[237,239],[241,241],[243,244],[246,252],[255,255],[258,259],[272, +273],[338,339],[376,376],[416,417],[431,432],[768,769],[771,771],[777,777],[803,803], +[8363,8363]],"cp1133":[[0,160],[162,162],[166,166],[172,172],[3713,3714],[3716,3716], +[3719,3720],[3722,3722],[3725,3725],[3732,3735],[3737,3743],[3745,3747],[3749,3749], +[3751,3751],[3754,3755],[3757,3769],[3771,3773],[3776,3780],[3782,3782],[3784,3789], +[3792,3801],[3804,3805],[8365,8365]],"cp1161":[[0,127],[160,160],[162,162],[166,166], +[172,172],[3585,3642],[3647,3675],[8364,8364]],"cp1162":[[0,127],[129,132],[134,144], +[152,160],[3585,3642],[3647,3675],[8211,8212],[8216,8217],[8220,8221],[8226,8226], +[8230,8230],[8364,8364]],"cp1163":[[0,163],[165,167],[169,179],[181,183],[185,194], +[196,203],[205,207],[209,209],[211,212],[214,220],[223,226],[228,235],[237,239],[241, +241],[243,244],[246,252],[255,255],[258,259],[272,273],[338,339],[376,376],[416,417], +[431,432],[768,769],[771,771],[777,777],[803,803],[8363,8364]],"cp437":[[0,127],[160, +163],[165,165],[170,172],[176,178],[181,181],[183,183],[186,189],[191,191],[196,199], +[201,201],[209,209],[214,214],[220,220],[223,226],[228,239],[241,244],[246,247],[249, +252],[255,255],[402,402],[915,915],[920,920],[931,931],[934,934],[937,937],[945,945], +[948,949],[960,960],[963,964],[966,966],[8319,8319],[8359,8359],[8729,8730],[8734, +8734],[8745,8745],[8776,8776],[8801,8801],[8804,8805],[8976,8976],[8992,8993],[9472, +9472],[9474,9474],[9484,9484],[9488,9488],[9492,9492],[9496,9496],[9500,9500],[9508, +9508],[9516,9516],[9524,9524],[9532,9532],[9552,9580],[9600,9600],[9604,9604],[9608, +9608],[9612,9612],[9616,9619],[9632,9632]],"cp737":[[0,127],[160,160],[176,178],[183, +183],[247,247],[902,902],[904,906],[908,908],[910,911],[913,929],[931,943],[945,974], +[8319,8319],[8729,8730],[8776,8776],[8804,8805],[9472,9472],[9474,9474],[9484,9484], +[9488,9488],[9492,9492],[9496,9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524], +[9532,9532],[9552,9580],[9600,9600],[9604,9604],[9608,9608],[9612,9612],[9616,9619], +[9632,9632]],"cp775":[[0,127],[160,160],[162,164],[166,167],[169,169],[171,174],[176, +179],[181,183],[185,185],[187,190],[196,198],[201,201],[211,211],[213,216],[220,220], +[223,223],[228,230],[233,233],[243,243],[245,248],[252,252],[256,257],[260,263],[268, +269],[274,275],[278,281],[290,291],[298,299],[302,303],[310,311],[315,316],[321,326], +[332,333],[342,343],[346,347],[352,353],[362,363],[370,371],[377,382],[8217,8217], +[8220,8222],[8729,8729],[9472,9472],[9474,9474],[9484,9484],[9488,9488],[9492,9492], +[9496,9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532,9532],[9552,9553], +[9556,9556],[9559,9559],[9562,9562],[9565,9565],[9568,9568],[9571,9571],[9574,9574], +[9577,9577],[9580,9580],[9600,9600],[9604,9604],[9608,9608],[9612,9612],[9616,9619], +[9632,9632]],"cp808":[[0,127],[160,160],[176,176],[183,183],[1025,1025],[1028,1028], +[1031,1031],[1038,1038],[1040,1103],[1105,1105],[1108,1108],[1111,1111],[1118,1118], +[8364,8364],[8470,8470],[8729,8730],[9472,9472],[9474,9474],[9484,9484],[9488,9488], +[9492,9492],[9496,9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532,9532], +[9552,9580],[9600,9600],[9604,9604],[9608,9608],[9612,9612],[9616,9619],[9632,9632]], +"cp850":[[0,127],[160,255],[305,305],[402,402],[8215,8215],[9472,9472],[9474,9474], +[9484,9484],[9488,9488],[9492,9492],[9496,9496],[9500,9500],[9508,9508],[9516,9516], +[9524,9524],[9532,9532],[9552,9553],[9556,9556],[9559,9559],[9562,9562],[9565,9565], +[9568,9568],[9571,9571],[9574,9574],[9577,9577],[9580,9580],[9600,9600],[9604,9604], +[9608,9608],[9617,9619],[9632,9632]],"cp852":[[0,127],[160,160],[164,164],[167,168], +[171,173],[176,176],[180,180],[184,184],[187,187],[193,194],[196,196],[199,199],[201, +201],[203,203],[205,206],[211,212],[214,215],[218,218],[220,221],[223,223],[225,226], +[228,228],[231,231],[233,233],[235,235],[237,238],[243,244],[246,247],[250,250],[252, +253],[258,263],[268,273],[280,283],[313,314],[317,318],[321,324],[327,328],[336,337], +[340,341],[344,347],[350,357],[366,369],[377,382],[711,711],[728,729],[731,731],[733, +733],[9472,9472],[9474,9474],[9484,9484],[9488,9488],[9492,9492],[9496,9496],[9500, +9500],[9508,9508],[9516,9516],[9524,9524],[9532,9532],[9552,9553],[9556,9556],[9559, +9559],[9562,9562],[9565,9565],[9568,9568],[9571,9571],[9574,9574],[9577,9577],[9580, +9580],[9600,9600],[9604,9604],[9608,9608],[9617,9619],[9632,9632]],"cp855":[[0,127], +[160,160],[164,164],[167,167],[171,171],[173,173],[187,187],[1025,1036],[1038,1103], +[1105,1116],[1118,1119],[8470,8470],[9472,9472],[9474,9474],[9484,9484],[9488,9488], +[9492,9492],[9496,9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532,9532], +[9552,9553],[9556,9556],[9559,9559],[9562,9562],[9565,9565],[9568,9568],[9571,9571], +[9574,9574],[9577,9577],[9580,9580],[9600,9600],[9604,9604],[9608,9608],[9617,9619], +[9632,9632]],"cp856":[[0,127],[160,160],[162,169],[171,185],[187,190],[215,215],[247, +247],[1488,1514],[8215,8215],[9472,9472],[9474,9474],[9484,9484],[9488,9488],[9492, +9492],[9496,9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532,9532],[9552, +9553],[9556,9556],[9559,9559],[9562,9562],[9565,9565],[9568,9568],[9571,9571],[9574, +9574],[9577,9577],[9580,9580],[9600,9600],[9604,9604],[9608,9608],[9617,9619],[9632, +9632]],"cp857":[[0,127],[160,207],[209,220],[223,239],[241,252],[255,255],[286,287], +[304,305],[350,351],[9472,9472],[9474,9474],[9484,9484],[9488,9488],[9492,9492],[9496, +9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532,9532],[9552,9553],[9556, +9556],[9559,9559],[9562,9562],[9565,9565],[9568,9568],[9571,9571],[9574,9574],[9577, +9577],[9580,9580],[9600,9600],[9604,9604],[9608,9608],[9617,9619],[9632,9632]],"cp858":[[0, +127],[160,255],[402,402],[8215,8215],[8364,8364],[9472,9472],[9474,9474],[9484,9484], +[9488,9488],[9492,9492],[9496,9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524], +[9532,9532],[9552,9553],[9556,9556],[9559,9559],[9562,9562],[9565,9565],[9568,9568], +[9571,9571],[9574,9574],[9577,9577],[9580,9580],[9600,9600],[9604,9604],[9608,9608], +[9617,9619],[9632,9632]],"cp860":[[0,127],[160,163],[170,172],[176,178],[181,181], +[183,183],[186,189],[191,195],[199,202],[204,205],[209,213],[217,218],[220,220],[223, +227],[231,234],[236,237],[241,245],[247,247],[249,250],[252,252],[915,915],[920,920], +[931,931],[934,934],[937,937],[945,945],[948,949],[960,960],[963,964],[966,966],[8319, +8319],[8359,8359],[8729,8730],[8734,8734],[8745,8745],[8776,8776],[8801,8801],[8804, +8805],[8992,8993],[9472,9472],[9474,9474],[9484,9484],[9488,9488],[9492,9492],[9496, +9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532,9532],[9552,9580],[9600, +9600],[9604,9604],[9608,9608],[9612,9612],[9616,9619],[9632,9632]],"cp861":[[0,127], +[160,161],[163,163],[171,172],[176,178],[181,181],[183,183],[187,189],[191,191],[193, +193],[196,199],[201,201],[205,205],[208,208],[211,211],[214,214],[216,216],[218,218], +[220,226],[228,235],[237,237],[240,240],[243,244],[246,248],[250,254],[402,402],[915, +915],[920,920],[931,931],[934,934],[937,937],[945,945],[948,949],[960,960],[963,964], +[966,966],[8319,8319],[8359,8359],[8729,8730],[8734,8734],[8745,8745],[8776,8776], +[8801,8801],[8804,8805],[8976,8976],[8992,8993],[9472,9472],[9474,9474],[9484,9484], +[9488,9488],[9492,9492],[9496,9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524], +[9532,9532],[9552,9580],[9600,9600],[9604,9604],[9608,9608],[9612,9612],[9616,9619], +[9632,9632]],"cp862":[[0,127],[160,163],[165,165],[170,172],[176,178],[181,181],[183, +183],[186,189],[191,191],[209,209],[223,223],[225,225],[237,237],[241,241],[243,243], +[247,247],[250,250],[402,402],[915,915],[920,920],[931,931],[934,934],[937,937],[945, +945],[948,949],[960,960],[963,964],[966,966],[1488,1514],[8319,8319],[8359,8359], +[8729,8730],[8734,8734],[8745,8745],[8776,8776],[8801,8801],[8804,8805],[8976,8976], +[8992,8993],[9472,9472],[9474,9474],[9484,9484],[9488,9488],[9492,9492],[9496,9496], +[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532,9532],[9552,9580],[9600,9600], +[9604,9604],[9608,9608],[9612,9612],[9616,9619],[9632,9632]],"cp863":[[0,127],[160, +160],[162,164],[166,168],[171,172],[175,184],[187,190],[192,192],[194,194],[199,203], +[206,207],[212,212],[217,217],[219,220],[223,224],[226,226],[231,235],[238,239],[243, +244],[247,247],[249,252],[402,402],[915,915],[920,920],[931,931],[934,934],[937,937], +[945,945],[948,949],[960,960],[963,964],[966,966],[8215,8215],[8319,8319],[8729,8730], +[8734,8734],[8745,8745],[8776,8776],[8801,8801],[8804,8805],[8976,8976],[8992,8993], +[9472,9472],[9474,9474],[9484,9484],[9488,9488],[9492,9492],[9496,9496],[9500,9500], +[9508,9508],[9516,9516],[9524,9524],[9532,9532],[9552,9580],[9600,9600],[9604,9604], +[9608,9608],[9612,9612],[9616,9619],[9632,9632]],"cp864":[[0,36],[38,127],[160,160], +[162,164],[166,166],[171,173],[176,177],[183,183],[187,189],[215,215],[247,247],[946, +946],[966,966],[1548,1548],[1563,1563],[1567,1567],[1600,1600],[1617,1617],[1632, +1642],[8729,8730],[8734,8734],[8776,8776],[9472,9472],[9474,9474],[9484,9484],[9488, +9488],[9492,9492],[9496,9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532, +9532],[9618,9618],[9632,9632],[63101,63101],[63104,63109],[63115,63115],[63117,63119], +[63121,63121],[63123,63123],[63125,63125],[63127,63127],[63129,63129],[63131,63131], +[63133,63133],[63135,63135],[63137,63137],[63139,63139],[63141,63141],[63143,63143], +[63145,63145],[63147,63147],[63149,63149],[63151,63151],[63153,63153],[63155,63155], +[63157,63157],[63159,63159],[63161,63161],[63163,63163],[63165,63165],[63167,63167], +[63169,63169],[63173,63173],[63177,63185],[63187,63187],[63189,63189],[63191,63191], +[63193,63193],[63195,63195],[63197,63197],[63199,63199],[63201,63201],[63203,63203], +[63205,63205],[63207,63207],[63209,63209],[63211,63213],[63215,63219],[63221,63224], +[63227,63228]],"cp865":[[0,127],[160,161],[163,164],[170,172],[176,178],[181,181], +[183,183],[186,186],[188,189],[191,191],[196,199],[201,201],[209,209],[214,214],[216, +216],[220,220],[223,226],[228,239],[241,244],[246,252],[255,255],[402,402],[915,915], +[920,920],[931,931],[934,934],[937,937],[945,945],[948,949],[960,960],[963,964],[966, +966],[8319,8319],[8359,8359],[8729,8730],[8734,8734],[8745,8745],[8776,8776],[8801, +8801],[8804,8805],[8976,8976],[8992,8993],[9472,9472],[9474,9474],[9484,9484],[9488, +9488],[9492,9492],[9496,9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532, +9532],[9552,9580],[9600,9600],[9604,9604],[9608,9608],[9612,9612],[9616,9619],[9632, +9632]],"cp866":[[0,127],[160,160],[164,164],[176,176],[183,183],[1025,1025],[1028, +1028],[1031,1031],[1038,1038],[1040,1103],[1105,1105],[1108,1108],[1111,1111],[1118, +1118],[8470,8470],[8729,8730],[9472,9472],[9474,9474],[9484,9484],[9488,9488],[9492, +9492],[9496,9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532,9532],[9552, +9580],[9600,9600],[9604,9604],[9608,9608],[9612,9612],[9616,9619],[9632,9632]],"cp869":[[0, +127],[160,160],[163,163],[166,169],[171,173],[176,179],[183,183],[187,187],[189,189], +[900,902],[904,906],[908,908],[910,929],[931,974],[8213,8213],[8216,8217],[9472,9472], +[9474,9474],[9484,9484],[9488,9488],[9492,9492],[9496,9496],[9500,9500],[9508,9508], +[9516,9516],[9524,9524],[9532,9532],[9552,9553],[9556,9556],[9559,9559],[9562,9562], +[9565,9565],[9568,9568],[9571,9571],[9574,9574],[9577,9577],[9580,9580],[9600,9600], +[9604,9604],[9608,9608],[9617,9619],[9632,9632]],"cp922":[[0,174],[176,207],[209, +221],[223,239],[241,253],[255,255],[352,353],[381,382],[8254,8254]],"cp936":[[0,127], +[164,164],[167,168],[176,177],[183,183],[215,215],[224,225],[232,234],[236,237],[242, +243],[247,247],[249,250],[252,252],[257,257],[275,275],[283,283],[299,299],[324,324], +[328,328],[333,333],[363,363],[462,462],[464,464],[466,466],[468,468],[470,470],[472, +472],[474,474],[476,476],[593,593],[609,609],[711,711],[713,715],[729,729],[913,929], +[931,937],[945,961],[963,969],[1025,1025],[1040,1103],[1105,1105],[8208,8208],[8211, +8214],[8216,8217],[8220,8221],[8229,8230],[8240,8240],[8242,8243],[8245,8245],[8251, +8251],[8364,8364],[8451,8451],[8453,8453],[8457,8457],[8470,8470],[8481,8481],[8544, +8555],[8560,8569],[8592,8595],[8598,8601],[8712,8712],[8719,8719],[8721,8721],[8725, +8725],[8730,8730],[8733,8736],[8739,8739],[8741,8741],[8743,8747],[8750,8750],[8756, +8759],[8765,8765],[8776,8776],[8780,8780],[8786,8786],[8800,8801],[8804,8807],[8814, +8815],[8853,8853],[8857,8857],[8869,8869],[8895,8895],[8978,8978],[9312,9321],[9332, +9371],[9472,9547],[9552,9587],[9601,9615],[9619,9621],[9632,9633],[9650,9651],[9660, +9661],[9670,9671],[9675,9675],[9678,9679],[9698,9701],[9733,9734],[9737,9737],[9792, +9792],[9794,9794],[12288,12291],[12293,12311],[12317,12318],[12321,12329],[12353, +12435],[12443,12446],[12449,12534],[12540,12542],[12549,12585],[12832,12841],[12849, +12849],[12963,12963],[13198,13199],[13212,13214],[13217,13217],[13252,13252],[13262, +13262],[13265,13266],[13269,13269],[19968,40869],[61740,61740],[61817,61817],[61845, +61845],[61927,61927],[61937,61937],[61964,61967],[61969,61969],[61971,61972],[61976, +61976],[61983,61985],[61987,61988],[61991,61993],[63024,63025],[63027,63044],[63049, +63058],[63060,63063],[63065,63078],[63080,63083],[63233,63326],[63456,63461]],"cp949":[[0, +127],[161,161],[164,164],[167,168],[170,170],[173,174],[176,180],[182,186],[188,191], +[198,198],[208,208],[215,216],[222,223],[230,230],[240,240],[247,248],[254,254],[273, +273],[294,295],[305,307],[312,312],[319,322],[329,331],[338,339],[358,359],[711,711], +[720,720],[728,731],[733,733],[913,929],[931,937],[945,961],[963,969],[1025,1025], +[1040,1103],[1105,1105],[8213,8213],[8216,8217],[8220,8221],[8224,8225],[8229,8230], +[8240,8240],[8242,8243],[8251,8251],[8308,8308],[8319,8319],[8321,8324],[8364,8364], +[8451,8451],[8457,8457],[8467,8467],[8470,8470],[8481,8482],[8486,8486],[8491,8491], +[8531,8532],[8539,8542],[8544,8553],[8560,8569],[8592,8601],[8658,8658],[8660,8660], +[8704,8704],[8706,8707],[8711,8712],[8715,8715],[8719,8719],[8721,8721],[8730,8730], +[8733,8734],[8736,8736],[8741,8741],[8743,8748],[8750,8750],[8756,8757],[8764,8765], +[8786,8786],[8800,8801],[8804,8805],[8810,8811],[8834,8835],[8838,8839],[8857,8857], +[8869,8869],[8978,8978],[9312,9326],[9332,9346],[9372,9397],[9424,9449],[9472,9475], +[9484,9547],[9618,9618],[9632,9633],[9635,9641],[9650,9651],[9654,9655],[9660,9661], +[9664,9665],[9670,9672],[9675,9675],[9678,9681],[9733,9734],[9742,9743],[9756,9756], +[9758,9758],[9792,9792],[9794,9794],[9824,9825],[9827,9829],[9831,9834],[9836,9837], +[12288,12291],[12296,12305],[12307,12309],[12353,12435],[12449,12534],[12593,12686], +[12800,12828],[12896,12923],[12927,12927],[13184,13188],[13192,13258],[13263,13264], +[13267,13267],[13270,13270],[13272,13272],[13275,13277],[19968,19969],[19971,19971], +[19975,19979],[19981,19981],[19985,19985],[19988,19990],[19992,19993],[19998,19998], +[20013,20013],[20018,20018],[20024,20025],[20027,20027],[20034,20035],[20037,20037], +[20043,20043],[20045,20047],[20054,20054],[20056,20057],[20061,20063],[20075,20075], +[20077,20077],[20083,20083],[20086,20087],[20094,20094],[20098,20098],[20102,20102], +[20104,20104],[20107,20108],[20110,20110],[20112,20114],[20116,20117],[20120,20120], +[20123,20123],[20126,20126],[20129,20130],[20132,20134],[20136,20136],[20139,20142], +[20150,20150],[20154,20154],[20160,20161],[20164,20164],[20167,20167],[20170,20171], +[20173,20173],[20180,20185],[20189,20189],[20191,20191],[20195,20197],[20208,20208], +[20210,20210],[20214,20215],[20219,20219],[20225,20225],[20233,20235],[20237,20241], +[20271,20271],[20276,20276],[20278,20278],[20280,20280],[20282,20282],[20284,20285], +[20291,20291],[20294,20296],[20301,20305],[20309,20309],[20313,20316],[20329,20329], +[20335,20336],[20339,20339],[20342,20342],[20346,20346],[20350,20351],[20353,20353], +[20355,20356],[20358,20358],[20360,20360],[20362,20363],[20365,20365],[20367,20367], +[20369,20369],[20374,20374],[20376,20376],[20379,20379],[20381,20381],[20398,20399], +[20405,20406],[20415,20415],[20418,20420],[20425,20426],[20430,20430],[20433,20433], +[20435,20436],[20439,20439],[20442,20442],[20445,20445],[20447,20449],[20462,20463], +[20465,20465],[20467,20467],[20469,20469],[20472,20472],[20474,20474],[20482,20482], +[20486,20486],[20489,20489],[20491,20491],[20493,20493],[20497,20498],[20502,20502], +[20505,20506],[20508,20508],[20510,20511],[20513,20513],[20515,20516],[20518,20520], +[20522,20525],[20539,20539],[20547,20547],[20551,20553],[20559,20559],[20565,20565], +[20570,20570],[20572,20572],[20581,20581],[20596,20598],[20600,20600],[20608,20608], +[20613,20613]],"cp950":[[0,127],[167,167],[175,177],[183,183],[215,215],[247,247], +[711,711],[713,715],[717,717],[729,729],[913,929],[931,937],[945,961],[963,969],[8211, +8212],[8216,8217],[8220,8221],[8229,8231],[8242,8242],[8245,8245],[8251,8251],[8364, +8364],[8451,8451],[8453,8453],[8457,8457],[8544,8553],[8592,8595],[8598,8601],[8725, +8725],[8730,8730],[8734,8736],[8739,8739],[8741,8741],[8745,8747],[8750,8750],[8756, +8757],[8786,8786],[8800,8801],[8806,8807],[8853,8853],[8857,8857],[8869,8869],[8895, +8895],[9472,9472],[9474,9474],[9484,9484],[9488,9488],[9492,9492],[9496,9496],[9500, +9500],[9508,9508],[9516,9516],[9524,9524],[9532,9532],[9552,9588],[9601,9615],[9619, +9621],[9632,9633],[9650,9651],[9660,9661],[9670,9671],[9675,9675],[9678,9679],[9698, +9701],[9733,9734],[9792,9792],[9794,9794],[12288,12291],[12296,12306],[12308,12309], +[12317,12318],[12321,12329],[12549,12585],[12963,12963],[13198,13199],[13212,13214], +[13217,13217],[13252,13252],[13262,13262],[13265,13266],[13269,13269],[19968,19969], +[19971,19971],[19975,19985],[19988,19990],[19992,19993],[19998,19999],[20006,20006], +[20011,20011],[20013,20014],[20016,20019],[20024,20025],[20027,20028],[20034,20035], +[20037,20037],[20039,20040],[20043,20043],[20045,20047],[20050,20051],[20054,20054], +[20056,20057],[20060,20063],[20073,20073],[20083,20083],[20094,20095],[20098,20100], +[20102,20102],[20104,20104],[20107,20110],[20113,20117],[20121,20121],[20123,20123], +[20126,20127],[20129,20130],[20132,20134],[20136,20136],[20139,20142],[20147,20147], +[20150,20150],[20153,20154],[20160,20164],[20166,20171],[20173,20173],[20180,20186], +[20188,20191],[20193,20193],[20195,20197],[20200,20201],[20208,20215],[20219,20219], +[20221,20221],[20223,20226],[20228,20229],[20232,20235],[20237,20245],[20248,20249], +[20253,20253],[20258,20258],[20268,20269],[20271,20272],[20275,20276],[20278,20278], +[20280,20280],[20282,20287],[20289,20289],[20291,20291],[20294,20297],[20300,20324], +[20327,20327],[20329,20332],[20334,20336],[20339,20361],[20363,20363],[20365,20365], +[20367,20370],[20372,20376],[20378,20382],[20398,20399],[20402,20403],[20405,20407], +[20409,20411],[20415,20421],[20423,20423],[20425,20427],[20429,20433],[20435,20436], +[20438,20449],[20460,20460],[20462,20463],[20465,20465],[20467,20472],[20474,20474], +[20478,20478],[20480,20480],[20485,20487],[20489,20489],[20491,20495],[20497,20508], +[20510,20515],[20517,20525],[20527,20529],[20531,20531],[20533,20533],[20535,20535], +[20540,20540],[20544,20545],[20547,20547],[20549,20559],[20561,20561],[20563,20563], +[20565,20565],[20567,20567],[20570,20581],[20584,20587],[20589,20592],[20594,20599], +[20602,20602],[20605,20605],[20608,20608],[20610,20611],[20613,20613],[20615,20615], +[20619,20622],[20625,20626],[20628,20630],[20632,20638],[20642,20643],[20652,20664], +[20666,20667],[20669,20671],[20673,20674],[20676,20683],[20686,20687],[20689,20689], +[20691,20695],[20698,20699],[20701,20701],[20704,20704],[20707,20714],[20716,20721], +[20723,20723],[20725,20726],[20728,20729],[20731,20731],[20733,20736],[20738,20748], +[20752,20757],[20759,20760],[20762,20762],[20764,20764],[20767,20770],[20772,20774], +[20777,20778],[20781,20782],[20784,20789],[20791,20797],[20799,20801],[20803,20809], +[20811,20813],[20818,20818],[20820,20821],[20823,20823],[20825,20831],[20833,20835], +[20837,20837],[20839,20841],[20843,20846],[20849,20849],[20853,20856],[20860,20860], +[20864,20864],[20871,20871],[20873,20874],[20877,20877],[20879,20879],[20881,20885]], +"eucjp":[[0,127],[161,161],[164,170],[174,177],[180,180],[182,182],[184,184],[186, +186],[191,207],[209,275],[278,290],[292,299],[302,333],[336,382],[461,476],[501,501], +[711,711],[728,731],[733,733],[900,902],[904,906],[908,908],[910,929],[931,974],[1025, +1036],[1038,1103],[1105,1116],[1118,1119],[8208,8208],[8213,8213],[8216,8217],[8220, +8221],[8224,8225],[8229,8230],[8240,8240],[8242,8243],[8251,8251],[8254,8254],[8451, +8451],[8470,8470],[8481,8482],[8491,8491],[8544,8553],[8560,8569],[8592,8595],[8658, +8658],[8660,8660],[8704,8704],[8706,8707],[8711,8712],[8715,8715],[8721,8721],[8730, +8730],[8733,8736],[8741,8741],[8743,8748],[8750,8750],[8756,8757],[8765,8765],[8786, +8786],[8800,8801],[8806,8807],[8810,8811],[8834,8835],[8838,8839],[8869,8869],[8895, +8895],[8978,8978],[9312,9331],[9472,9475],[9484,9484],[9487,9488],[9491,9492],[9495, +9496],[9499,9501],[9504,9504],[9507,9509],[9512,9512],[9515,9516],[9519,9520],[9523, +9524],[9527,9528],[9531,9532],[9535,9535],[9538,9538],[9547,9547],[9632,9633],[9650, +9651],[9660,9661],[9670,9671],[9675,9675],[9678,9679],[9711,9711],[9733,9734],[9792, +9792],[9794,9794],[9834,9834],[9837,9837],[9839,9839],[12288,12291],[12293,12309], +[12317,12317],[12319,12319],[12353,12435],[12443,12446],[12449,12534],[12539,12542], +[12849,12850],[12857,12857],[12964,12968],[13059,13059],[13069,13069],[13076,13076], +[13080,13080],[13090,13091],[13094,13095],[13099,13099],[13110,13110],[13115,13115], +[13129,13130],[13133,13133],[13137,13137],[13143,13143],[13179,13182],[13198,13199], +[13212,13214],[13217,13217],[13252,13252],[13261,13261],[19968,19973],[19975,19982], +[19984,19986],[19988,19993],[19998,19999],[20001,20001],[20003,20004],[20006,20006], +[20008,20008],[20010,20011],[20013,20018],[20021,20022],[20024,20025],[20027,20028], +[20031,20037],[20039,20039],[20043,20043],[20045,20047],[20049,20049],[20053,20058], +[20060,20063],[20066,20067],[20072,20073],[20081,20081],[20083,20085],[20089,20089], +[20094,20096],[20098,20098],[20101,20102],[20104,20110],[20113,20114],[20116,20121], +[20123,20130],[20132,20134],[20136,20136],[20139,20144],[20147,20147],[20150,20150], +[20153,20154],[20160,20164],[20166,20167],[20170,20171],[20173,20176],[20180,20187], +[20189,20197],[20200,20200],[20205,20211],[20213,20215],[20219,20227],[20232,20242], +[20245,20247],[20249,20250],[20252,20253],[20270,20273],[20275,20286],[20288,20288], +[20290,20291],[20294,20297],[20299,20320],[20323,20323],[20329,20330],[20332,20332], +[20334,20337],[20339,20339],[20341,20351],[20353,20358],[20360,20372],[20374,20379], +[20381,20385],[20395,20395],[20397,20399],[20402,20402],[20405,20407],[20409,20409], +[20411,20422],[20424,20434],[20436,20436],[20439,20440],[20442,20445],[20447,20453], +[20462,20464],[20466,20467],[20469,20470],[20472,20472],[20474,20474],[20476,20481], +[20484,20487],[20489,20500],[20502,20511],[20513,20526],[20528,20528],[20530,20531], +[20533,20534],[20537,20537],[20539,20539],[20544,20547],[20549,20554],[20556,20556], +[20558,20563],[20565,20567],[20569,20570],[20572,20572],[20575,20576],[20578,20579], +[20581,20583],[20586,20586],[20588,20589],[20592,20594],[20596,20598],[20600,20600], +[20605,20605],[20608,20609],[20611,20614],[20618,20618],[20621,20628],[20630,20630], +[20632,20636],[20638,20642],[20650,20650],[20652,20653],[20655,20656],[20658,20661], +[20663,20663],[20665,20666],[20669,20670],[20672,20672],[20674,20677]],"gb18030":[[0, +2160639]],"gbk":[[0,127],[164,164],[167,168],[176,177],[183,183],[215,215],[224,225], +[232,234],[236,237],[242,243],[247,247],[249,250],[252,252],[257,257],[275,275],[283, +283],[299,299],[324,324],[328,328],[333,333],[363,363],[462,462],[464,464],[466,466], +[468,468],[470,470],[472,472],[474,474],[476,476],[505,505],[593,593],[609,609],[711, +711],[713,715],[729,729],[913,929],[931,937],[945,961],[963,969],[1025,1025],[1040, +1103],[1105,1105],[8208,8208],[8211,8214],[8216,8217],[8220,8221],[8229,8230],[8240, +8240],[8242,8243],[8245,8245],[8251,8251],[8364,8364],[8451,8451],[8453,8453],[8457, +8457],[8470,8470],[8481,8481],[8544,8555],[8560,8569],[8592,8595],[8598,8601],[8712, +8712],[8719,8719],[8721,8721],[8725,8725],[8730,8730],[8733,8736],[8739,8739],[8741, +8741],[8743,8747],[8750,8750],[8756,8759],[8765,8765],[8776,8776],[8780,8780],[8786, +8786],[8800,8801],[8804,8807],[8814,8815],[8853,8853],[8857,8857],[8869,8869],[8895, +8895],[8978,8978],[9312,9321],[9332,9371],[9472,9547],[9552,9587],[9601,9615],[9619, +9621],[9632,9633],[9650,9651],[9660,9661],[9670,9671],[9675,9675],[9678,9679],[9698, +9701],[9733,9734],[9737,9737],[9792,9792],[9794,9794],[11905,11905],[11908,11908], +[11912,11912],[11915,11916],[11927,11927],[11943,11943],[11946,11946],[11950,11950], +[11955,11955],[11958,11959],[11963,11963],[11978,11978],[12272,12283],[12288,12291], +[12293,12311],[12317,12318],[12321,12329],[12350,12350],[12353,12435],[12443,12446], +[12449,12534],[12540,12542],[12549,12585],[12832,12841],[12849,12849],[12963,12963], +[13198,13199],[13212,13214],[13217,13217],[13252,13252],[13262,13262],[13265,13266], +[13269,13269],[13383,13383],[13427,13427],[13726,13726],[13838,13838],[13850,13850], +[14616,14616],[14702,14702],[14799,14800],[14815,14815],[14963,14963],[15182,15182], +[15470,15470],[15584,15584],[16470,16470],[16735,16735],[17207,17207],[17324,17324], +[17329,17329],[17373,17373],[17622,17622],[17996,17996],[18017,18017],[18211,18211], +[18217,18217],[18300,18300],[18317,18317],[18759,18759],[18810,18810],[18813,18813], +[18818,18819],[18821,18822],[18843,18843],[18847,18847],[18870,18871],[19575,19575], +[19615,19619],[19731,19737],[19886,19886],[19968,40869],[55296,56804],[56806,57195], +[57197,57287],[57289,57318],[57332,57364],[57366,57368],[57374,57374],[57382,57382], +[57387,57388],[57393,57394],[57403,57403],[57411,57411],[57428,57429],[57444,57444], +[61740,61740],[61817,61817],[61845,61845],[61927,61927],[61937,61937],[61964,61967], +[61969,61969],[61971,61972],[61976,61976],[61983,61985],[61987,61988],[61991,61993], +[63024,63025],[63027,63044],[63049,63058],[63060,63063],[63065,63078],[63080,63083], +[63233,63326],[63456,63461]],"georgianacademy":[[0,129],[141,144],[157,158],[160, +191],[231,255],[338,339],[352,353],[376,376],[402,402],[710,710],[732,732],[4304, +4342],[8211,8212],[8216,8218],[8220,8222],[8224,8226],[8230,8230],[8240,8240],[8249, +8250],[8482,8482]],"georgianps":[[0,129],[141,144],[157,158],[160,191],[230,255], +[338,339],[352,353],[376,376],[402,402],[710,710],[732,732],[4304,4341],[8211,8212], +[8216,8218],[8220,8222],[8224,8226],[8230,8230],[8240,8240],[8249,8250],[8482,8482]], +"hex":[],"hproman8":[[0,165],[167,168],[170,171],[175,177],[180,183],[186,214],[216, +246],[248,255],[352,353],[376,376],[402,402],[710,710],[715,715],[732,732],[8212, +8212],[8356,8356],[9632,9632]],"iso646cn":[[0,35],[37,125],[127,127],[165,165],[8254, +8254]],"iso646jp":[[0,91],[93,125],[127,127],[165,165],[8254,8254]],"iso88591":[[0, +255]],"iso885910":[[0,160],[167,167],[173,173],[176,176],[183,183],[193,198],[201, +201],[203,203],[205,208],[211,214],[216,216],[218,223],[225,230],[233,233],[235,235], +[237,240],[243,246],[248,248],[250,254],[256,257],[260,261],[268,269],[272,275],[278, +281],[290,291],[296,299],[302,303],[310,312],[315,316],[325,326],[330,333],[352,353], +[358,363],[370,371],[381,382],[8213,8213]],"iso885911":[[0,160],[3585,3642],[3647, +3675]],"iso885913":[[0,160],[162,164],[166,167],[169,169],[171,174],[176,179],[181, +183],[185,185],[187,190],[196,198],[201,201],[211,211],[213,216],[220,220],[223,223], +[228,230],[233,233],[243,243],[245,248],[252,252],[256,257],[260,263],[268,269],[274, +275],[278,281],[290,291],[298,299],[302,303],[310,311],[315,316],[321,326],[332,333], +[342,343],[346,347],[352,353],[362,363],[370,371],[377,382],[8217,8217],[8220,8222]], +"iso885914":[[0,160],[163,163],[167,167],[169,169],[173,174],[182,182],[192,207], +[209,214],[216,221],[223,239],[241,246],[248,253],[255,255],[266,267],[288,289],[372, +376],[7682,7683],[7690,7691],[7710,7711],[7744,7745],[7766,7767],[7776,7777],[7786, +7787],[7808,7813],[7922,7923]],"iso885915":[[0,163],[165,165],[167,167],[169,179], +[181,183],[185,187],[191,255],[338,339],[352,353],[376,376],[381,382],[8364,8364]], +"iso885916":[[0,160],[167,167],[169,169],[171,171],[173,173],[176,177],[182,183], +[187,187],[192,194],[196,196],[198,207],[210,212],[214,214],[217,220],[223,226],[228, +228],[230,239],[242,244],[246,246],[249,252],[255,255],[258,263],[268,269],[272,273], +[280,281],[321,324],[336,339],[346,347],[352,353],[368,369],[376,382],[536,539],[8221, +8222],[8364,8364]],"iso88592":[[0,160],[164,164],[167,168],[173,173],[176,176],[180, +180],[184,184],[193,194],[196,196],[199,199],[201,201],[203,203],[205,206],[211,212], +[214,215],[218,218],[220,221],[223,223],[225,226],[228,228],[231,231],[233,233],[235, +235],[237,238],[243,244],[246,247],[250,250],[252,253],[258,263],[268,273],[280,283], +[313,314],[317,318],[321,324],[327,328],[336,337],[340,341],[344,347],[350,357],[366, +369],[377,382],[711,711],[728,729],[731,731],[733,733]],"iso88593":[[0,160],[163, +164],[167,168],[173,173],[176,176],[178,181],[183,184],[189,189],[192,194],[196,196], +[199,207],[209,212],[214,215],[217,220],[223,226],[228,228],[231,239],[241,244],[246, +247],[249,252],[264,267],[284,289],[292,295],[304,305],[308,309],[348,351],[364,365], +[379,380],[728,729]],"iso88594":[[0,160],[164,164],[167,168],[173,173],[175,176], +[180,180],[184,184],[193,198],[201,201],[203,203],[205,206],[212,216],[218,220],[223, +223],[225,230],[233,233],[235,235],[237,238],[244,248],[250,252],[256,257],[260,261], +[268,269],[272,275],[278,281],[290,291],[296,299],[302,303],[310,312],[315,316],[325, +326],[330,333],[342,343],[352,353],[358,363],[370,371],[381,382],[711,711],[729,729], +[731,731]],"iso88595":[[0,160],[167,167],[173,173],[1025,1036],[1038,1103],[1105, +1116],[1118,1119],[8470,8470]],"iso88596":[[0,160],[164,164],[173,173],[1548,1548], +[1563,1563],[1567,1567],[1569,1594],[1600,1618]],"iso88597":[[0,160],[163,163],[166, +169],[171,173],[176,179],[183,183],[187,187],[189,189],[890,890],[900,902],[904,906], +[908,908],[910,929],[931,974],[8213,8213],[8216,8217],[8364,8364],[8367,8367]],"iso88598":[[0, +160],[162,169],[171,185],[187,190],[215,215],[247,247],[1488,1514],[8206,8207],[8215, +8215]],"iso88599":[[0,207],[209,220],[223,239],[241,252],[255,255],[286,287],[304, +305],[350,351]],"koi8r":[[0,127],[160,160],[169,169],[176,176],[178,178],[183,183], +[247,247],[1025,1025],[1040,1103],[1105,1105],[8729,8730],[8776,8776],[8804,8805], +[8992,8993],[9472,9472],[9474,9474],[9484,9484],[9488,9488],[9492,9492],[9496,9496], +[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532,9532],[9552,9580],[9600,9600], +[9604,9604],[9608,9608],[9612,9612],[9616,9619],[9632,9632]],"koi8ru":[[0,127],[160, +160],[169,169],[176,176],[178,178],[183,183],[247,247],[1025,1025],[1028,1028],[1030, +1031],[1038,1038],[1040,1103],[1105,1105],[1108,1108],[1110,1111],[1118,1118],[1168, +1169],[8729,8730],[8776,8776],[8804,8805],[8992,8993],[9472,9472],[9474,9474],[9484, +9484],[9488,9488],[9492,9492],[9496,9496],[9500,9500],[9508,9508],[9516,9516],[9524, +9524],[9532,9532],[9552,9554],[9556,9556],[9559,9563],[9566,9569],[9571,9571],[9574, +9578],[9600,9600],[9604,9604],[9608,9608],[9612,9612],[9616,9619],[9632,9632]],"koi8t":[[0, +127],[164,164],[166,167],[169,169],[171,174],[176,178],[182,183],[187,187],[1025, +1025],[1040,1103],[1105,1105],[1170,1171],[1178,1179],[1202,1203],[1206,1207],[1250, +1251],[1262,1263],[8211,8212],[8216,8218],[8220,8222],[8224,8226],[8230,8230],[8240, +8240],[8249,8250],[8470,8470],[8482,8482]],"koi8u":[[0,127],[160,160],[169,169],[176, +176],[178,178],[183,183],[247,247],[1025,1025],[1028,1028],[1030,1031],[1040,1103], +[1105,1105],[1108,1108],[1110,1111],[1168,1169],[8729,8730],[8776,8776],[8804,8805], +[8992,8993],[9472,9472],[9474,9474],[9484,9484],[9488,9488],[9492,9492],[9496,9496], +[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532,9532],[9552,9554],[9556,9556], +[9559,9563],[9565,9569],[9571,9571],[9574,9578],[9580,9580],[9600,9600],[9604,9604], +[9608,9608],[9612,9612],[9616,9619],[9632,9632]],"maccenteuro":[[0,127],[160,160], +[163,163],[167,169],[171,172],[174,174],[176,176],[182,182],[187,187],[193,193],[196, +196],[201,201],[205,205],[211,214],[218,218],[220,221],[223,223],[225,225],[228,228], +[233,233],[237,237],[243,247],[250,250],[252,253],[256,257],[260,263],[268,271],[274, +275],[278,283],[290,291],[298,299],[302,303],[310,311],[313,318],[321,328],[332,333], +[336,337],[340,347],[352,353],[356,357],[362,363],[366,371],[377,382],[711,711],[8211, +8212],[8216,8218],[8220,8222],[8224,8224],[8226,8226],[8230,8230],[8249,8250],[8482, +8482],[8706,8706],[8710,8710],[8721,8721],[8730,8730],[8800,8800],[8804,8805],[9674, +9674]],"maccroatian":[[0,127],[160,164],[167,172],[174,177],[180,184],[186,187],[191, +207],[209,214],[216,220],[223,239],[241,252],[262,263],[268,269],[272,273],[305,305], +[338,339],[352,353],[381,382],[402,402],[710,711],[730,730],[732,732],[960,960],[8211, +8212],[8216,8218],[8220,8222],[8224,8224],[8226,8226],[8230,8230],[8240,8240],[8249, +8250],[8260,8260],[8482,8482],[8486,8486],[8706,8706],[8710,8710],[8719,8719],[8721, +8721],[8730,8730],[8734,8734],[8747,8747],[8776,8776],[8800,8800],[8804,8805],[9674, +9674]],"maccyrillic":[[0,127],[160,160],[162,164],[167,167],[169,169],[171,172],[174, +174],[176,177],[181,182],[187,187],[247,247],[402,402],[1025,1036],[1038,1103],[1105, +1116],[1118,1119],[8211,8212],[8216,8217],[8220,8222],[8224,8224],[8226,8226],[8230, +8230],[8470,8470],[8482,8482],[8706,8706],[8710,8710],[8730,8730],[8734,8734],[8776, +8776],[8800,8800],[8804,8805]],"macgreek":[[0,127],[160,160],[163,163],[165,169], +[171,174],[176,179],[185,185],[187,187],[189,189],[196,196],[201,201],[214,214],[220, +220],[223,224],[226,226],[228,228],[231,235],[238,239],[244,244],[246,247],[249,249], +[251,252],[339,339],[900,906],[908,908],[910,929],[931,974],[8211,8211],[8213,8213], +[8216,8217],[8220,8221],[8224,8224],[8226,8226],[8230,8230],[8240,8240],[8482,8482], +[8776,8776],[8800,8800],[8804,8805]],"maciceland":[[0,127],[160,165],[167,172],[174, +177],[180,184],[186,187],[191,214],[216,255],[305,305],[338,339],[376,376],[402,402], +[710,711],[728,733],[960,960],[8211,8212],[8216,8218],[8220,8222],[8226,8226],[8230, +8230],[8240,8240],[8260,8260],[8482,8482],[8486,8486],[8706,8706],[8710,8710],[8719, +8719],[8721,8721],[8730,8730],[8734,8734],[8747,8747],[8776,8776],[8800,8800],[8804, +8805],[9674,9674]],"macintosh":[[0,127],[160,165],[167,172],[174,177],[180,184],[186, +187],[191,207],[209,214],[216,220],[223,239],[241,252],[255,255],[305,305],[338,339], +[376,376],[402,402],[710,711],[728,733],[960,960],[8211,8212],[8216,8218],[8220,8222], +[8224,8226],[8230,8230],[8240,8240],[8249,8250],[8260,8260],[8482,8482],[8486,8486], +[8706,8706],[8710,8710],[8719,8719],[8721,8721],[8730,8730],[8734,8734],[8747,8747], +[8776,8776],[8800,8800],[8804,8805],[9674,9674],[62209,62210]],"macroman":[[0,127], +[160,165],[167,172],[174,177],[180,184],[186,187],[191,207],[209,214],[216,220],[223, +239],[241,252],[255,255],[305,305],[338,339],[376,376],[402,402],[710,711],[728,733], +[960,960],[8211,8212],[8216,8218],[8220,8222],[8224,8226],[8230,8230],[8240,8240], +[8249,8250],[8260,8260],[8482,8482],[8486,8486],[8706,8706],[8710,8710],[8719,8719], +[8721,8721],[8730,8730],[8734,8734],[8747,8747],[8776,8776],[8800,8800],[8804,8805], +[9674,9674],[62209,62210]],"macromania":[[0,127],[160,165],[167,172],[174,177],[180, +184],[186,187],[191,197],[199,207],[209,214],[217,220],[223,229],[231,239],[241,247], +[249,252],[255,255],[258,259],[305,305],[338,339],[350,351],[354,355],[376,376],[402, +402],[710,711],[728,733],[960,960],[8211,8212],[8216,8218],[8220,8222],[8224,8226], +[8230,8230],[8240,8240],[8249,8250],[8260,8260],[8482,8482],[8486,8486],[8706,8706], +[8710,8710],[8719,8719],[8721,8721],[8730,8730],[8734,8734],[8747,8747],[8776,8776], +[8800,8800],[8804,8805],[9674,9674]],"macthai":[[0,127],[160,160],[169,169],[171, +171],[174,174],[187,187],[3585,3642],[3647,3661],[3663,3673],[8203,8203],[8211,8212], +[8216,8217],[8220,8221],[8226,8226],[8230,8230],[8482,8482],[61572,61593],[63231, +63231]],"macturkish":[[0,127],[160,163],[165,165],[167,172],[174,177],[180,184],[186, +187],[191,207],[209,214],[216,220],[223,239],[241,252],[255,255],[286,287],[304,305], +[338,339],[350,351],[376,376],[402,402],[710,711],[728,733],[960,960],[8211,8212], +[8216,8218],[8220,8222],[8224,8226],[8230,8230],[8240,8240],[8482,8482],[8486,8486], +[8706,8706],[8710,8710],[8719,8719],[8721,8721],[8730,8730],[8734,8734],[8747,8747], +[8776,8776],[8800,8800],[8804,8805],[9674,9674]],"macukraine":[[0,127],[160,160], +[163,164],[167,167],[169,169],[171,172],[174,174],[176,177],[181,182],[187,187],[247, +247],[402,402],[1025,1036],[1038,1103],[1105,1116],[1118,1119],[1168,1169],[8211, +8212],[8216,8217],[8220,8222],[8224,8224],[8226,8226],[8230,8230],[8470,8470],[8482, +8482],[8710,8710],[8730,8730],[8734,8734],[8776,8776],[8800,8800],[8804,8805]],"mik":[[0, +127],[160,160],[167,167],[176,178],[181,181],[183,183],[223,223],[247,247],[915,915], +[920,920],[931,931],[934,934],[937,937],[945,945],[948,949],[960,960],[963,964],[966, +966],[1040,1103],[8319,8319],[8470,8470],[8729,8730],[8734,8734],[8745,8745],[8776, +8776],[8801,8801],[8804,8805],[8992,8993],[9472,9472],[9474,9474],[9484,9484],[9488, +9488],[9492,9492],[9496,9496],[9500,9500],[9508,9508],[9516,9516],[9524,9524],[9532, +9532],[9552,9553],[9556,9556],[9559,9559],[9562,9562],[9565,9565],[9568,9568],[9571, +9571],[9574,9574],[9577,9577],[9580,9580],[9600,9600],[9604,9604],[9608,9608],[9612, +9612],[9616,9619],[9632,9632]],"pt154":[[0,127],[160,160],[167,167],[169,169],[171, +172],[174,174],[176,176],[182,183],[187,187],[1025,1025],[1030,1030],[1032,1032], +[1038,1038],[1040,1103],[1105,1105],[1110,1110],[1112,1112],[1118,1118],[1170,1171], +[1174,1181],[1184,1187],[1194,1195],[1198,1203],[1206,1211],[1240,1241],[1250,1251], +[1256,1257],[1262,1263],[8211,8212],[8216,8217],[8220,8222],[8226,8226],[8230,8230], +[8470,8470]],"rk1048":[[0,127],[160,160],[164,164],[166,167],[169,169],[171,174], +[176,177],[181,183],[187,187],[1025,1027],[1030,1030],[1033,1034],[1039,1103],[1105, +1107],[1110,1110],[1113,1114],[1119,1119],[1170,1171],[1178,1179],[1186,1187],[1198, +1201],[1210,1211],[1240,1241],[1256,1257],[8211,8212],[8216,8218],[8220,8222],[8224, +8226],[8230,8230],[8240,8240],[8249,8250],[8364,8364],[8470,8470],[8482,8482]],"shiftjis":[[0, +128],[165,165],[167,168],[176,177],[180,180],[182,182],[215,215],[247,247],[913,929], +[931,937],[945,961],[963,969],[1025,1025],[1040,1103],[1105,1105],[8208,8208],[8213, +8213],[8216,8217],[8220,8221],[8224,8225],[8229,8230],[8240,8240],[8242,8243],[8251, +8251],[8254,8254],[8451,8451],[8470,8470],[8481,8481],[8491,8491],[8544,8553],[8560, +8569],[8592,8595],[8658,8658],[8660,8660],[8704,8704],[8706,8707],[8711,8712],[8715, +8715],[8721,8721],[8730,8730],[8733,8736],[8741,8741],[8743,8748],[8750,8750],[8756, +8757],[8765,8765],[8786,8786],[8800,8801],[8806,8807],[8810,8811],[8834,8835],[8838, +8839],[8869,8869],[8895,8895],[8978,8978],[9312,9331],[9472,9475],[9484,9484],[9487, +9488],[9491,9492],[9495,9496],[9499,9501],[9504,9504],[9507,9509],[9512,9512],[9515, +9516],[9519,9520],[9523,9524],[9527,9528],[9531,9532],[9535,9535],[9538,9538],[9547, +9547],[9632,9633],[9650,9651],[9660,9661],[9670,9671],[9675,9675],[9678,9679],[9711, +9711],[9733,9734],[9792,9792],[9794,9794],[9834,9834],[9837,9837],[9839,9839],[12288, +12291],[12293,12309],[12317,12317],[12319,12319],[12353,12435],[12443,12446],[12449, +12534],[12539,12542],[12849,12850],[12857,12857],[12964,12968],[13059,13059],[13069, +13069],[13076,13076],[13080,13080],[13090,13091],[13094,13095],[13099,13099],[13110, +13110],[13115,13115],[13129,13130],[13133,13133],[13137,13137],[13143,13143],[13179, +13182],[13198,13199],[13212,13214],[13217,13217],[13252,13252],[13261,13261],[19968, +19969],[19971,19971],[19975,19979],[19981,19982],[19984,19985],[19988,19993],[19998, +19998],[20001,20001],[20006,20006],[20008,20008],[20010,20010],[20013,20013],[20017, +20018],[20022,20022],[20024,20025],[20027,20028],[20031,20031],[20034,20035],[20037, +20037],[20043,20043],[20045,20047],[20053,20057],[20061,20063],[20066,20066],[20081, +20081],[20083,20083],[20094,20094],[20096,20096],[20098,20098],[20101,20102],[20104, +20108],[20110,20110],[20113,20114],[20116,20117],[20120,20121],[20123,20124],[20126, +20130],[20132,20134],[20136,20136],[20139,20142],[20144,20144],[20147,20147],[20150, +20150],[20154,20154],[20160,20162],[20164,20164],[20166,20167],[20170,20171],[20173, +20175],[20180,20185],[20189,20191],[20193,20193],[20195,20197],[20205,20206],[20208, +20208],[20210,20210],[20214,20215],[20219,20220],[20224,20225],[20227,20227],[20233, +20234],[20237,20241],[20250,20250],[20252,20253],[20271,20272],[20276,20276],[20278, +20278],[20280,20282],[20284,20285],[20291,20291],[20294,20295],[20301,20305],[20307, +20307],[20309,20311],[20313,20318],[20329,20329],[20335,20336],[20339,20339],[20341, +20342],[20347,20348],[20351,20351],[20355,20355],[20358,20358],[20360,20360],[20362, +20363],[20365,20365],[20367,20367],[20369,20370],[20372,20372],[20374,20374],[20376, +20376],[20378,20379],[20381,20381],[20384,20385],[20395,20395],[20397,20399],[20405, +20406],[20415,20415],[20418,20420],[20425,20426],[20429,20430],[20432,20433],[20436, +20436],[20439,20440],[20442,20443],[20445,20445],[20447,20447],[20449,20449],[20451, +20453],[20462,20463],[20467,20467],[20469,20470],[20472,20472],[20474,20474],[20478, +20479],[20485,20486],[20489,20489],[20491,20491],[20493,20493],[20495,20495],[20497, +20498],[20500,20500],[20502,20502],[20505,20506],[20510,20511],[20513,20518],[20520, +20525],[20534,20534],[20537,20537],[20544,20544],[20546,20547],[20550,20553],[20559, +20560],[20565,20566],[20570,20570],[20572,20572],[20581,20581],[20588,20588],[20592, +20592],[20594,20594]],"tcvn":[[3,3],[7,16],[24,127],[160,160],[192,195],[200,202], +[204,205],[210,213],[217,218],[221,221],[224,227],[232,234],[236,237],[242,245],[249, +250],[253,253],[258,259],[272,273],[296,297],[360,361],[416,417],[431,432],[768,769], +[771,771],[777,777],[803,803],[7840,7929]],"tis620":[[0,127],[3585,3642],[3647,3675]], +"ucs2":[[0,2160639]],"ucs4":[[0,2160639]],"utf16":[[0,2160639]],"utf16be":[[0,2160639]], +"utf32":[[0,2160639]],"utf32be":[[0,2160639]],"utf32le":[[0,2160639]],"utf7":[[0, +2160639]],"utf7imap":[[0,2160639]],"utf8":[[0,2160639]],"viscii":[[0,1],[3,4],[7, +19],[21,24],[26,29],[31,127],[192,195],[200,202],[204,205],[210,213],[217,218],[221, +221],[224,227],[232,234],[236,237],[242,245],[249,250],[253,253],[258,259],[272,273], +[296,297],[360,361],[416,417],[431,432],[7840,7929]],"windows1250":[[0,127],[160, +160],[164,164],[166,169],[171,174],[176,177],[180,184],[187,187],[193,194],[196,196], +[199,199],[201,201],[203,203],[205,206],[211,212],[214,215],[218,218],[220,221],[223, +223],[225,226],[228,228],[231,231],[233,233],[235,235],[237,238],[243,244],[246,247], +[250,250],[252,253],[258,263],[268,273],[280,283],[313,314],[317,318],[321,324],[327, +328],[336,337],[340,341],[344,347],[350,357],[366,369],[377,382],[711,711],[728,729], +[731,731],[733,733],[8211,8212],[8216,8218],[8220,8222],[8224,8226],[8230,8230],[8240, +8240],[8249,8250],[8364,8364],[8482,8482]],"windows1251":[[0,127],[160,160],[164, +164],[166,167],[169,169],[171,174],[176,177],[181,183],[187,187],[1025,1036],[1038, +1103],[1105,1116],[1118,1119],[1168,1169],[8211,8212],[8216,8218],[8220,8222],[8224, +8226],[8230,8230],[8240,8240],[8249,8250],[8364,8364],[8470,8470],[8482,8482]],"windows1252":[[0, +127],[160,255],[338,339],[352,353],[376,376],[381,382],[402,402],[710,710],[732,732], +[8211,8212],[8216,8218],[8220,8222],[8224,8226],[8230,8230],[8240,8240],[8249,8250], +[8364,8364],[8482,8482]],"windows1253":[[0,127],[160,160],[163,169],[171,174],[176, +179],[181,183],[187,187],[189,189],[402,402],[900,902],[904,906],[908,908],[910,929], +[931,974],[8211,8213],[8216,8218],[8220,8222],[8224,8226],[8230,8230],[8240,8240], +[8249,8250],[8364,8364],[8482,8482]],"windows1254":[[0,127],[160,207],[209,220],[223, +239],[241,252],[255,255],[286,287],[304,305],[338,339],[350,353],[376,376],[402,402], +[710,710],[732,732],[8211,8212],[8216,8218],[8220,8222],[8224,8226],[8230,8230],[8240, +8240],[8249,8250],[8364,8364],[8482,8482]],"windows1255":[[0,127],[160,163],[165, +169],[171,185],[187,191],[215,215],[247,247],[402,402],[710,710],[732,732],[1456, +1475],[1488,1514],[1520,1524],[8206,8207],[8211,8212],[8216,8218],[8220,8222],[8224, +8226],[8230,8230],[8240,8240],[8249,8250],[8362,8362],[8364,8364],[8482,8482]],"windows1256":[[0, +127],[160,160],[162,169],[171,185],[187,190],[215,215],[224,224],[226,226],[231,235], +[238,239],[244,244],[247,247],[249,249],[251,252],[338,339],[402,402],[710,710],[1548, +1548],[1563,1563],[1567,1567],[1569,1594],[1600,1618],[1657,1657],[1662,1662],[1670, +1670],[1672,1672],[1681,1681],[1688,1688],[1705,1705],[1711,1711],[1722,1722],[1726, +1726],[1729,1729],[1746,1746],[8204,8207],[8211,8212],[8216,8218],[8220,8222],[8224, +8226],[8230,8230],[8240,8240],[8249,8250],[8364,8364],[8482,8482]],"windows1257":[[0, +127],[160,160],[162,164],[166,169],[171,185],[187,190],[196,198],[201,201],[211,211], +[213,216],[220,220],[223,223],[228,230],[233,233],[243,243],[245,248],[252,252],[256, +257],[260,263],[268,269],[274,275],[278,281],[290,291],[298,299],[302,303],[310,311], +[315,316],[321,326],[332,333],[342,343],[346,347],[352,353],[362,363],[370,371],[377, +382],[711,711],[729,729],[731,731],[8211,8212],[8216,8218],[8220,8222],[8224,8226], +[8230,8230],[8240,8240],[8249,8250],[8364,8364],[8482,8482]],"windows1258":[[0,127], +[160,194],[196,203],[205,207],[209,209],[211,212],[214,220],[223,226],[228,235],[237, +239],[241,241],[243,244],[246,252],[255,255],[258,259],[272,273],[338,339],[376,376], +[402,402],[416,417],[431,432],[710,710],[732,732],[768,769],[771,771],[777,777],[803, +803],[8211,8212],[8216,8218],[8220,8222],[8224,8226],[8230,8230],[8240,8240],[8249, +8250],[8363,8364],[8482,8482]],"windows874":[[0,127],[160,160],[3585,3642],[3647, +3675],[8211,8212],[8216,8217],[8220,8221],[8226,8226],[8230,8230],[8364,8364]]} diff --git a/generation/gen-transliteration.js b/generation/gen-transliteration.js new file mode 100644 index 00000000..064219ee --- /dev/null +++ b/generation/gen-transliteration.js @@ -0,0 +1,59 @@ +var iconv = require('../lib/index'); +var fs = require('fs'); + +// Make sure we can tell missing characters apart from question marks. +iconv.defaultCharSingleByte = '\u0000'; + +// Trigger loading of encodings +iconv.getCodec('ascii'); + +var encodings = Object.keys(iconv.encodings).filter(function(enc) { + return enc.charAt(0) !== '_' && typeof iconv.encodings[enc] !== 'string'; +}).sort(); + +var allCharsStr = ''; +var skip = 0; + +for (var i = 0; i <= 0x10F7FF; ++i) { + if (i === 0xD800) + skip = 0x800; + + allCharsStr += String.fromCodePoint(i + skip); +} + +var encodingRanges = {}; + +encodings.forEach(function(enc) { + var chars = iconv.decode(iconv.encode(allCharsStr, enc), enc); + var ranges = []; + var start = -1; + + // Yes, less-than-or-equal, deliberately going one extra character beyond the end of the string. + for (var i = 0; i <= chars.length; ++i) { + var cp = chars.codePointAt(i); + + var badChar = (isNaN(cp) || cp === 0 || cp === 0xFFFD); + + if (i !== 0 && i !== 0xF7FD && i !== 0xFFFD) { + if (start < 0 && !badChar) + start = (i === 1 ? 0 : i); + else if (start >= 0 && badChar) { + ranges.push([start, i - 1]); + start = -1; + + if (ranges.length > 255) + break; + } + } + + if (cp > 0xFFFF) + ++i; + } + + encodingRanges[enc] = ranges; +}); + +var json = JSON.stringify(encodingRanges); + +json = json.replace(/(.{80}.*?(,|$))/g, '$1\n') + '\n'; +fs.writeFileSync(__dirname + '/../encodings/tables/transliteration-ranges.json', json); diff --git a/lib/index.d.ts b/lib/index.d.ts index 0547eb34..e6c2c0f7 100644 --- a/lib/index.d.ts +++ b/lib/index.d.ts @@ -6,19 +6,27 @@ *--------------------------------------------------------------------------------------------*/ declare module 'iconv-lite' { - export function decode(buffer: Buffer, encoding: string, options?: Options): string; + export function decode(buffer: Buffer, encoding: string, options?: Options): string; - export function encode(content: string, encoding: string, options?: Options): Buffer; + export function encode(content: string, encoding: string, options?: Options): Buffer; - export function encodingExists(encoding: string): boolean; + export function encodingExists(encoding: string): boolean; - export function decodeStream(encoding: string, options?: Options): NodeJS.ReadWriteStream; + export function decodeStream(encoding: string, options?: Options): NodeJS.ReadWriteStream; - export function encodeStream(encoding: string, options?: Options): NodeJS.ReadWriteStream; + export function encodeStream(encoding: string, options?: Options): NodeJS.ReadWriteStream; + + export function transliterate(str: string, targetEncoding: string, options?: TransliterationOptions): string; } -export interface Options { - stripBOM?: boolean; +export interface TransliterationOptions { + german?: boolean; + smartSpacing?: boolean; +} + +export interface Options extends TransliterationOptions { addBOM?: boolean; defaultEncoding?: string; + stripBOM?: boolean; + transliterate?: boolean; } diff --git a/lib/index.js b/lib/index.js index 5391919c..f05dba12 100644 --- a/lib/index.js +++ b/lib/index.js @@ -18,12 +18,22 @@ iconv.defaultCharSingleByte = '?'; // Public API. iconv.encode = function encode(str, encoding, options) { str = "" + (str || ""); // Ensure string. + var parts = (typeof encoding === 'string' ? encoding.toLowerCase().split('//') : null); + + if (parts && parts.length > 1) { + encoding = parts[0]; + + if (parts[1] === 'translit') { + options = options || {}; + options.transliterate = true; + } + } var encoder = iconv.getEncoder(encoding, options); var res = encoder.write(str); var trail = encoder.end(); - + return (trail && trail.length > 0) ? Buffer.concat([res, trail]) : res; } @@ -63,9 +73,10 @@ iconv._codecDataCache = {}; iconv.getCodec = function getCodec(encoding) { if (!iconv.encodings) iconv.encodings = require("../encodings"); // Lazy load all encoding definitions. - + // Canonicalize encoding name: strip all non-alphanumeric chars and appended year. var enc = iconv._canonicalizeEncoding(encoding); + var rootName = enc; // Traverse iconv.encodings to find actual codec. var codecOptions = {}; @@ -78,7 +89,7 @@ iconv.getCodec = function getCodec(encoding) { switch (typeof codecDef) { case "string": // Direct alias to other encoding. - enc = codecDef; + enc = rootName = codecDef; break; case "object": // Alias with options. Can be layered. @@ -87,7 +98,7 @@ iconv.getCodec = function getCodec(encoding) { if (!codecOptions.encodingName) codecOptions.encodingName = enc; - + enc = codecDef.type; break; @@ -99,6 +110,9 @@ iconv.getCodec = function getCodec(encoding) { // It'll be called only once (for each different options object). codec = new codecDef(codecOptions, iconv); + if (!codec.encodingName) + codec.encodingName = rootName; + iconv._codecDataCache[codecOptions.encodingName] = codec; // Save it to be reused later. return codec; @@ -117,9 +131,14 @@ iconv.getEncoder = function getEncoder(encoding, options) { var codec = iconv.getCodec(encoding), encoder = new codec.encoder(options, codec); + encoder.encodingName = codec.encodingName; + if (codec.bomAware && options && options.addBOM) encoder = new bomHandling.PrependBOM(encoder, options); + if (options && options.transliterate) + encoder = new TransliterationWrapper(encoder, options); + return encoder; } @@ -133,6 +152,192 @@ iconv.getDecoder = function getDecoder(encoding, options) { return decoder; } +String.prototype.codePointAt || (String.prototype.codePointAt = function(index) { + var code = this.charCodeAt(index); + + if (!isNaN(code) && 0xD800 <= code && code <= 0xDBFF) { + var surr = this.charCodeAt(index + 1); + + if (!isNaN(surr) && 0xDC00 <= surr && surr <= 0xDFFF) + code = 0x10000 + ((code - 0xD800) << 10) + (surr - 0xDC00); + } + + return code; +}); + +function TransliterationWrapper(encoder, options) { + this.encoder = encoder; + this.encodingName = encoder.encodingName + '-translit'; + this.smartSpacing = (options && options.smartSpacing); + this.german = (options && options.german); + this.options = { deferredSmartSpacing: this.smartSpacing, german: this.german }; + this.pending = ''; +} + +var MAX_PENDING = 16384; + +TransliterationWrapper.prototype.write = function(str) { + var $; + + if (this.german && ($ = /^(.*)([aou])$/i.exec(this.pending))) { + str = $[2] + str; + this.pending = $[1]; + } + + str = iconv.transliterate(str, this.encoder.encodingName, this.options); + + if (this.smartSpacing) { + str = this.pending + str; + this.pending = ''; + + // Split the text being written out at a safe place where smart spacing won't + // need to make any changes. + $ = /^(.*)([^\s\x80\x81][\s\x80\x81].*)$/.exec(str); + + if ($) { + str = unidecode.resolveSpacing($[1]); + this.pending = $[2]; + } + else if (str.length > MAX_PENDING) + str = unidecode.resolveSpacing(str); + else if (str.length > 0 && /[^\x80\x81]/.test(str)) { + // Save one character from the end for the next spacing context + this.pending = str.charAt(str.length - 1); + str = str.substr(0, str.length - 1); + } + else { + this.pending = str; + str = ''; + } + } + + // If in German mode, and the last character currently available is an A, O, or U, make it a pending + // character on the chance the character that comes along next is a combining diaeresis. + if (this.german && ($ = /^(.*)([aou])$/i.exec(str))) { + str = $[1]; + this.pending = $[2] + this.pending; + } + + return this.encoder.write(str); +}; + +TransliterationWrapper.prototype.end = function() { + var buf = null; + var bufs = []; + + if (this.pending) { + buf = this.encoder.write(this.smartSpacing ? unidecode.resolveSpacing(this.pending) : this.pending); + this.pending = ''; + } + + if (buf) + bufs.push(buf); + + buf = this.encoder.end(); + + if (buf) + bufs.push(buf); + + return Buffer.concat(bufs); +}; + +var checkedForUnidecode = false; +var unidecode; +var hasUnidecodePlus = false; +var encodingRanges; +var translitWarnings = {}; +var ranges; +var plusOptions; +var codepoints; + +try { + // Can we use Unicode-aware regexes? + codepoints = /[^\x00-\x7F]/gu; +} +catch (err) { + // Nope! This mess will have to do. + codepoints = /(?:[\x80-\uD7FF\uE000-\uFFFF]|[\uD800-\uDBFF][\uDC00-\uDFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF])/g; +} + +iconv.transliterate = function transliterate(str, targetEncoding, options) { + var codec = iconv.getCodec(targetEncoding); + // Should throw an error before reaching here for an invalid encoding + var encoding = codec.encodingName; + + if (/^(utf.*|cesu8|gb18030|ucs2|ucs4)$/.test(encoding)) + return str; + + iconv.hasUnidecodePlus(); + + var deferredSmartSpacing = options && options.deferredSmartSpacing; + var smartSpacing = deferredSmartSpacing || (options && options.smartSpacing); + var german = options && options.german; + + if (!hasUnidecodePlus && (smartSpacing || german)) + throw new Error('Options "smartSpacing" and "german" are only available when using unidecode-plus'); + else if (hasUnidecodePlus) + plusOptions = { deferredSmartSpacing: deferredSmartSpacing, german: german, smartSpacing: smartSpacing }; + else + plusOptions = undefined; + + // For plain ASCII, we can do this quickly. + if (encoding === 'ascii') + return unidecode(str, plusOptions); + + if (!encodingRanges) + encodingRanges = require('../encodings/tables/transliteration-ranges.json'); + + ranges = encodingRanges[encoding]; + + if (!ranges) { + if (!translitWarnings[targetEncoding]) { + translitWarnings[targetEncoding] = true; + console.warning('Transliteration not available for "' + targetEncoding + '".'); + } + + return str; + } + + if (smartSpacing) { + options.skipRanges = ranges; + return unidecode(str, options); + } + else + // Much to my surprise, using a regex like this is much faster than building a string in a for loop. + return str.replace(codepoints, getTransliteration); +}; + +function getTransliteration(ch) { + var cp = ch.codePointAt(0); + + for (var j = 0; j < ranges.length; ++j) { + if (ranges[j][0] <= cp && cp <= ranges[j][1]) + return ch; + } + + return unidecode(ch); +} + +iconv.hasUnidecodePlus = function () { + if (!checkedForUnidecode) { + try { + unidecode = require('unidecode-plus'); + hasUnidecodePlus = true; + } + catch (err) { + try { + unidecode = require('unidecode'); + } + catch (err) { + throw new Error('Transliteration requires unidecode package'); + } + } + + checkedForUnidecode = true; + } + + return hasUnidecodePlus; +}; // Load extensions in Node. All of them are omitted in Browserify build via 'browser' field in package.json. var nodeVer = typeof process !== 'undefined' && process.versions && process.versions.node; diff --git a/package.json b/package.json index 513e1fa9..ea0ca5e5 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "iconv", "convert", "charset", - "icu" + "icu", + "transliterate" ], "author": "Alexander Shtuchkin ", "main": "./lib/index.js", @@ -24,21 +25,24 @@ "scripts": { "coverage": "istanbul cover _mocha -- --grep .", "coverage-open": "open coverage/lcov-report/index.html", - "test": "mocha --reporter spec --grep ." + "test": "mocha --reporter spec --grep .", + "translit": "node generation/gen-transliteration.js" }, "browser": { "./lib/extend-node": false, "./lib/streams": false }, "devDependencies": { - "mocha": "^3.1.0", - "request": "~2.87.0", - "unorm": "*", - "errto": "*", + "@types/node": "^12.0.10", "async": "*", + "errto": "*", + "iconv": "*", "istanbul": "*", + "mocha": "^3.1.0", + "request": "~2.87.0", "semver": "*", - "iconv": "*" + "unidecode-plus": "^1.0.2", + "unorm": "*" }, "dependencies": { "safer-buffer": ">= 2.1.2 < 3" diff --git a/test/translit-test.js b/test/translit-test.js new file mode 100644 index 00000000..0d4e4221 --- /dev/null +++ b/test/translit-test.js @@ -0,0 +1,101 @@ +var assert = require('assert'), + iconv = require(__dirname + '/../'), + Iconv = require('iconv').Iconv; + +var sample1 = '¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ_ØÙÚÛÜ_Þßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ\n\ +字符编码 文字コード Κωδικοποίηση χαρακτήρων Кодировка символов'; + +describe('Direct transliteration', function() { + it('should transliterate for ISO-8859-1 (as latin1)', function() { + assert.equal(iconv.transliterate(sample1, 'latin1'), '¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ_ØÙÚÛÜ_Þßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ\n\ +Zi Fu Bian Ma Wen Zi kodo Kodikopoiese kharakteron Kodirovka simvolov'); + }); + + it('should transliterate for ASCII', function() { + assert.equal(iconv.transliterate(sample1, 'ascii'), '!C/PS$?Y=|SS"(c)a<>1/41/23/4?AAAAAAAECEEEEIIIIDNOOOOO_OUUUU_Thssaaaaaaaeceeeeiiiidnooooo/ouuuuythy\n\ +Zi Fu Bian Ma Wen Zi kodo Kodikopoiese kharakteron Kodirovka simvolov'); + }); + + it('should transliterate for EUC-JP', function() { + assert.equal(iconv.transliterate(sample1, 'EUC-JP'), '¡C/PS¤¥¦§¨©ª<>1/41/23/4¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏDÑÒÓÔÕÖ_ØÙÚÛÜ_Þßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ\n\ +Zi Fu Bian Ma Wen Zi コード Κωδικοποίηση χαρακτήρων Кодировка символов'); + }); + + it('should transliterate for Windows-1251', function() { + assert.equal(iconv.transliterate(sample1, 'windows1251'), '!C/PS¤Y=¦§"©a«¬­®-°±23\'µ¶·,1o»1/41/23/4?AAAAAAAECEEEEIIIIDNOOOOO_OUUUU_Thssaaaaaaaeceeeeiiiidnooooo/ouuuuythy\n\ +Zi Fu Bian Ma Wen Zi kodo Kodikopoiese kharakteron Кодировка символов'); + }); + + it('should transliterate for MacGreek', function() { + assert.equal(iconv.transliterate(sample1, 'MacGreek'), '!C/£$?¥¦§¨©a«¬­®-°±²³\'uP*,¹o»1/4½3/4?AAAAÄAAECEÉEEIIIIDNOOOOÖ_OUUUÜ_Thßàaâaäaaeçèéêëiiîïdnooôoö÷oùuûüythy\n\ +Zi Fu Bian Ma Wen Zi kodo Κωδικοποίηση χαρακτήρων Kodirovka simvolov'); + }); +}); + +describe('Transliteration via encoder', function() { + it('should transliterate for ISO-8859-1 (as latin1)', function() { + var buf = iconv.encode(sample1, 'latin1//translit'); + assert.equal(iconv.decode(buf, 'latin1'), '¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ_ØÙÚÛÜ_Þßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ\n\ +Zi Fu Bian Ma Wen Zi kodo Kodikopoiese kharakteron Kodirovka simvolov'); + }); + + it('should transliterate for ASCII', function() { + var buf = iconv.encode(sample1, 'ascii', { transliterate: true }); + assert.equal(iconv.decode(buf, 'ascii'), '!C/PS$?Y=|SS"(c)a<>1/41/23/4?AAAAAAAECEEEEIIIIDNOOOOO_OUUUU_Thssaaaaaaaeceeeeiiiidnooooo/ouuuuythy\n\ +Zi Fu Bian Ma Wen Zi kodo Kodikopoiese kharakteron Kodirovka simvolov'); + }); + + // TODO: Put this test back when `¥` and `‾` are handled correctly. +// it('should transliterate for EUC-JP', function() { +// var buf = iconv.encode(sample1, 'EUC-JP', { transliterate: true }); +// assert.equal(iconv.decode(buf, 'EUC-JP'), '¡C/PS¤¥¦§¨©ª<>1/41/23/4¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏDÑÒÓÔÕÖ_ØÙÚÛÜ_Þßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ\n\ +// Zi Fu Bian Ma Wen Zi コード Κωδικοποίηση χαρακτήρων Кодировка символов'); +// }); + + it('should transliterate for Windows-1251', function() { + var buf = iconv.encode(sample1, 'windows1251', { transliterate: true }); + assert.equal(iconv.decode(buf, 'windows1251'), '!C/PS¤Y=¦§"©a«¬­®-°±23\'µ¶·,1o»1/41/23/4?AAAAAAAECEEEEIIIIDNOOOOO_OUUUU_Thssaaaaaaaeceeeeiiiidnooooo/ouuuuythy\n\ +Zi Fu Bian Ma Wen Zi kodo Kodikopoiese kharakteron Кодировка символов'); + }); + + it('should transliterate for MacGreek', function() { + var buf = iconv.encode(sample1, 'MacGreek', { transliterate: true }); + assert.equal(iconv.decode(buf, 'MacGreek'), '!C/£$?¥¦§¨©a«¬­®-°±²³\'uP*,¹o»1/4½3/4?AAAAÄAAECEÉEEIIIIDNOOOOÖ_OUUUÜ_Thßàaâaäaaeçèéêëiiîïdnooôoö÷oùuûüythy\n\ +Zi Fu Bian Ma Wen Zi kodo Κωδικοποίηση χαρακτήρων Kodirovka simvolov'); + }); + + if (iconv.hasUnidecodePlus()) { + it('should transliterate for ASCII with smart spacing', function() { + var buf = iconv.encode('Café 北京, 鞋 size 10½, 33⅓ RPM', 'ascii', { transliterate: true, smartSpacing: true }); + assert.equal(iconv.decode(buf, 'ascii'), 'Cafe Bei Jing, Xie size 10 1/2, 33 1/3 RPM'); + }); + + it('should transliterate for ISO-8859-1 with smart spacing', function() { + var buf = iconv.encode('Café 北京, 鞋 size 10½, 33⅓ RPM', 'ISO-8859-1', { transliterate: true, smartSpacing: true }); + assert.equal(iconv.decode(buf, 'ISO-8859-1'), 'Café Bei Jing, Xie size 10½, 33 1/3 RPM'); + }); + + it('should transliterate for ASCII with German option', function() { + var buf = iconv.encode('ÄäÖöÜü, Schrödinger', 'ascii', { transliterate: true, german: true }); + assert.equal(iconv.decode(buf, 'ascii'), 'AEaeOEoeUEue, Schroedinger'); + }); + } +}); + +describe('Transliteration via stream', function() { + it('should handle inconvenient breaks in spacing and accented characters', function() { + var encoder = iconv.getEncoder('ascii', { transliterate: true, smartSpacing: true, german: true }); + var buf = new Buffer([]); + + buf = Buffer.concat([buf, encoder.write('😁北Schro')]); + buf = Buffer.concat([buf, encoder.write('\u0308dinger❜s cat 10')]); + buf = Buffer.concat([buf, encoder.write('½')]); + + var end = encoder.end(); + + if (end) + buf = Buffer.concat([buf, end]); + + assert.equal(iconv.decode(buf, 'ascii'), ':-D Bei Schroedinger\'s cat 10 1/2'); + }); +}); diff --git a/test/utf32-test.js b/test/utf32-test.js index 23faab20..57c6e7d1 100644 --- a/test/utf32-test.js +++ b/test/utf32-test.js @@ -1,5 +1,5 @@ var assert = require('assert'), - iconv = require(__dirname+'/../'), + iconv = require(__dirname + '/../'), Iconv = require('iconv').Iconv; var testStr = '1aя中文☃💩',