sqlite3/ext/fts3/unicode/mkunicode.tcl

source [file join [file dirname [info script]] parseunicode.tcl]

proc print_rd {map} {
  global tl_lookup_table
  set aChar [list]
  set lRange [list]

  set nRange 1
  set iFirst  [lindex $map 0 0]
  set cPrev   [lindex $map 0 1]
  set fPrev   [lindex $map 0 2]

  foreach m [lrange $map 1 end] {
    foreach {i c f} $m {}

    if {$cPrev == $c && $fPrev==$f} {
      for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
        if {[info exists tl_lookup_table($j)]==0} break
      }

      if {$j==$i} {
        set nNew [expr {(1 + $i - $iFirst)}]
        if {$nNew<=8} {
          set nRange $nNew
          continue
        }
      }
    }

    lappend lRange [list $iFirst $nRange]
    lappend aChar  $cPrev
    lappend aFlag  $fPrev

    set iFirst $i
    set cPrev  $c
    set fPrev  $f
    set nRange 1
  }
  lappend lRange [list $iFirst $nRange]
  lappend aChar $cPrev
  lappend aFlag $fPrev

  puts "/*"
  puts "** If the argument is a codepoint corresponding to a lowercase letter"
  puts "** in the ASCII range with a diacritic added, return the codepoint"
  puts "** of the ASCII letter only. For example, if passed 235 - \"LATIN"
  puts "** SMALL LETTER E WITH DIAERESIS\" - return 65 (\"LATIN SMALL LETTER"
  puts "** E\"). The resuls of passing a codepoint that corresponds to an"
  puts "** uppercase letter are undefined."
  puts "*/"
  puts "static int ${::remove_diacritic}(int c, int bComplex)\{"
  puts "  unsigned short aDia\[\] = \{"
  puts -nonewline "        0, "
  set i 1
  foreach r $lRange {
    foreach {iCode nRange} $r {}
    if {($i % 8)==0} {puts "" ; puts -nonewline "    " }
    incr i

    puts -nonewline [format "%5d" [expr ($iCode<<3) + $nRange-1]]
    puts -nonewline ", "
  }
  puts ""
  puts "  \};"
  puts "#define HIBIT ((unsigned char)0x80)"
  puts "  unsigned char aChar\[\] = \{"
  puts -nonewline "    '\\0',      "
  set i 1
  foreach c $aChar f $aFlag {
    if { $f } {
      set str "'$c'|HIBIT, "
    } else {
      set str "'$c',       "
    }
    if {$c == ""} { set str "'\\0',      " }

    if {($i % 6)==0} {puts "" ; puts -nonewline "    " }
    incr i
    puts -nonewline "$str"
  }
  puts ""
  puts "  \};"
  puts {
  unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
  int iRes = 0;
  int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
  int iLo = 0;
  while( iHi>=iLo ){
    int iTest = (iHi + iLo) / 2;
    if( key >= aDia[iTest] ){
      iRes = iTest;
      iLo = iTest+1;
    }else{
      iHi = iTest-1;
    }
  }
  assert( key>=aDia[iRes] );
  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);}
  puts "\}"
}

proc print_isdiacritic {zFunc map} {

  set lCode [list]
  foreach m $map {
    foreach {code char flag} $m {}
    if {$flag} continue
    if {$code && $char == ""} { lappend lCode $code }
  }
  set lCode [lsort -integer $lCode]
  set iFirst [lindex $lCode 0]
  set iLast [lindex $lCode end]

  set i1 0
  set i2 0

  foreach c $lCode {
    set i [expr $c - $iFirst]
    if {$i < 32} {
      set i1 [expr {$i1 | (1<<$i)}]
    } else {
      set i2 [expr {$i2 | (1<<($i-32))}]
    }
  }

  puts "/*"
  puts "** Return true if the argument interpreted as a unicode codepoint" 
  puts "** is a diacritical modifier character."
  puts "*/"
  puts "int ${zFunc}\(int c)\{"
  puts "  unsigned int mask0 = [format "0x%08X" $i1];"
  puts "  unsigned int mask1 = [format "0x%08X" $i2];"

  puts "  if( c<$iFirst || c>$iLast ) return 0;"
  puts "  return (c < $iFirst+32) ?"
  puts "      (mask0 & ((unsigned int)1 << (c-$iFirst))) :"
  puts "      (mask1 & ((unsigned int)1 << (c-$iFirst-32)));"
  puts "\}"
}


#-------------------------------------------------------------------------

proc an_load_separator_ranges {} {
  global unicodedata.txt
  set lSep [an_load_unicodedata_text ${unicodedata.txt}]
  unset -nocomplain iFirst 
  unset -nocomplain nRange 
  set lRange [list]
  foreach sep $lSep {
    if {0==[info exists iFirst]} {
      set iFirst $sep
      set nRange 1
    } elseif { $sep == ($iFirst+$nRange) } {
      incr nRange
    } else {
      lappend lRange [list $iFirst $nRange]
      set iFirst $sep
      set nRange 1
    }
  } 
  lappend lRange [list $iFirst $nRange]
  set lRange
}

proc an_print_range_array {lRange} {
  set iFirstMax 0
  set nRangeMax 0
  foreach range $lRange {
    foreach {iFirst nRange} $range {}
    if {$iFirst > $iFirstMax} {set iFirstMax $iFirst}
    if {$nRange > $nRangeMax} {set nRangeMax $nRange}
  }
  if {$iFirstMax >= (1<<22)} {error "first-max is too large for format"}
  if {$nRangeMax >= (1<<10)} {error "range-max is too large for format"}

  puts -nonewline "  "
  puts [string trim {
  /* Each unsigned integer in the following array corresponds to a contiguous
  ** range of unicode codepoints that are not either letters or numbers (i.e.
  ** codepoints for which this function should return 0).
  **
  ** The most significant 22 bits in each 32-bit value contain the first 
  ** codepoint in the range. The least significant 10 bits are used to store
  ** the size of the range (always at least 1). In other words, the value 
  ** ((C<<22) + N) represents a range of N codepoints starting with codepoint 
  ** C. It is not possible to represent a range larger than 1023 codepoints 
  ** using this format.
  */
  }]
  puts -nonewline "  static const unsigned int aEntry\[\] = \{"
  set i 0
  foreach range $lRange {
    foreach {iFirst nRange} $range {}
    set u32 [format "0x%08X" [expr ($iFirst<<10) + $nRange]]

    if {($i % 5)==0} {puts "" ; puts -nonewline "   "}
    puts -nonewline " $u32,"
    incr i
  }
  puts ""
  puts "  \};"
}

proc an_print_ascii_bitmap {lRange} {
  foreach range $lRange {
    foreach {iFirst nRange} $range {}
    for {set i $iFirst} {$i < ($iFirst+$nRange)} {incr i} {
      if {$i<=127} { set a($i) 1 }
    }
  }

  set aAscii [list 0 0 0 0]
  foreach key [array names a] {
    set idx [expr $key >> 5]
    lset aAscii $idx [expr [lindex $aAscii $idx] | (1 << ($key&0x001F))]
  }

  puts "  static const unsigned int aAscii\[4\] = \{"
  puts -nonewline "   "
  foreach v $aAscii { puts -nonewline [format " 0x%08X," $v] }
  puts ""
  puts "  \};"
}

proc print_isalnum {zFunc lRange} {
  puts "/*"
  puts "** Return true if the argument corresponds to a unicode codepoint"
  puts "** classified as either a letter or a number. Otherwise false."
  puts "**"
  puts "** The results are undefined if the value passed to this function"
  puts "** is less than zero."
  puts "*/"
  puts "int ${zFunc}\(int c)\{"
  an_print_range_array $lRange
  an_print_ascii_bitmap $lRange
  puts {
  if( (unsigned int)c<128 ){
    return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
  }else if( (unsigned int)c<(1<<22) ){
    unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
    int iRes = 0;
    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
    int iLo = 0;
    while( iHi>=iLo ){
      int iTest = (iHi + iLo) / 2;
      if( key >= aEntry[iTest] ){
        iRes = iTest;
        iLo = iTest+1;
      }else{
        iHi = iTest-1;
      }
    }
    assert( aEntry[0]<key );
    assert( key>=aEntry[iRes] );
    return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
  }
  return 1;}
  puts "\}"
}

proc print_test_isalnum {zFunc lRange} {
  foreach range $lRange {
    foreach {iFirst nRange} $range {}
    for {set i $iFirst} {$i < ($iFirst+$nRange)} {incr i} { set a($i) 1 }
  }

  puts "static int isalnum_test(int *piCode)\{"
  puts -nonewline "  unsigned char aAlnum\[\] = \{"
  for {set i 0} {$i < 70000} {incr i} {
    if {($i % 32)==0} { puts "" ; puts -nonewline "    " }
    set bFlag [expr ![info exists a($i)]]
    puts -nonewline "${bFlag},"
  }
  puts ""
  puts "  \};"

  puts -nonewline "  int aLargeSep\[\] = \{"
  set i 0
  foreach iSep [lsort -integer [array names a]] {
    if {$iSep<70000} continue
    if {($i % 8)==0} { puts "" ; puts -nonewline "   " }
    puts -nonewline " $iSep,"
    incr i
  }
  puts ""
  puts "  \};"
  puts -nonewline "  int aLargeOther\[\] = \{"
  set i 0
  foreach iSep [lsort -integer [array names a]] {
    if {$iSep<70000} continue
    if {[info exists a([expr $iSep-1])]==0} {
      if {($i % 8)==0} { puts "" ; puts -nonewline "   " }
      puts -nonewline " [expr $iSep-1],"
      incr i
    }
    if {[info exists a([expr $iSep+1])]==0} {
      if {($i % 8)==0} { puts "" ; puts -nonewline "   " }
      puts -nonewline " [expr $iSep+1],"
      incr i
    }
  }
  puts ""
  puts "  \};"

  puts [subst -nocommands {
  int i;
  for(i=0; i<sizeof(aAlnum)/sizeof(aAlnum[0]); i++){
    if( ${zFunc}(i)!=aAlnum[i] ){
      *piCode = i;
      return 1;
    }
  }
  for(i=0; i<sizeof(aLargeSep)/sizeof(aLargeSep[0]); i++){
    if( ${zFunc}(aLargeSep[i])!=0 ){
      *piCode = aLargeSep[i];
      return 1;
    }
  }
  for(i=0; i<sizeof(aLargeOther)/sizeof(aLargeOther[0]); i++){
    if( ${zFunc}(aLargeOther[i])!=1 ){
      *piCode = aLargeOther[i];
      return 1;
    }
  }
  }]
  puts "  return 0;"
  puts "\}"
}

#-------------------------------------------------------------------------

proc tl_create_records {} {
  global tl_lookup_table

  set iFirst ""
  set nOff 0
  set nRange 0
  set nIncr 0

  set lRecord [list]
  foreach code [lsort -integer [array names tl_lookup_table]] {
    set mapping $tl_lookup_table($code)
    if {$iFirst == ""} {
      set iFirst $code
      set nOff   [expr $mapping - $code]
      set nRange 1
      set nIncr 1
    } else {
      set diff [expr $code - ($iFirst + ($nIncr * ($nRange - 1)))]
      if { $nRange==1 && ($diff==1 || $diff==2) } {
        set nIncr $diff
      }

      if {$diff != $nIncr || ($mapping - $code)!=$nOff} {
        if { $nRange==1 } {set nIncr 1}
        lappend lRecord [list $iFirst $nIncr $nRange $nOff]
        set iFirst $code
        set nOff   [expr $mapping - $code]
        set nRange 1
        set nIncr 1
      } else {
        incr nRange
      }
    }
  }

  lappend lRecord [list $iFirst $nIncr $nRange $nOff]

  set lRecord
}

proc tl_print_table_header {} {
  puts -nonewline "  "
  puts [string trim {
  /* Each entry in the following array defines a rule for folding a range
  ** of codepoints to lower case. The rule applies to a range of nRange
  ** codepoints starting at codepoint iCode.
  **
  ** If the least significant bit in flags is clear, then the rule applies
  ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
  ** need to be folded). Or, if it is set, then the rule only applies to
  ** every second codepoint in the range, starting with codepoint C.
  **
  ** The 7 most significant bits in flags are an index into the aiOff[]
  ** array. If a specific codepoint C does require folding, then its lower
  ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF).
  **
  ** The contents of this array are generated by parsing the CaseFolding.txt
  ** file distributed as part of the "Unicode Character Database". See
  ** http://www.unicode.org for details.
  */
  }]
  puts "  static const struct TableEntry \{"
  puts "    unsigned short iCode;"
  puts "    unsigned char flags;"
  puts "    unsigned char nRange;"
  puts "  \} aEntry\[\] = \{"
}

proc tl_print_table_entry {togglevar entry liOff} {
  upvar $togglevar t
  foreach {iFirst nIncr nRange nOff} $entry {}

  if {$iFirst > (1<<16)} { return 1 }

  if {[info exists t]==0} {set t 0}
  if {$t==0} { puts -nonewline "    " }

  set flags 0
  if {$nIncr==2} { set flags 1 ; set nRange [expr $nRange * 2]}
  if {$nOff<0}   { incr nOff [expr (1<<16)] }

  set idx [lsearch $liOff $nOff]
  if {$idx<0} {error "malfunction generating aiOff"}
  set flags [expr $flags + $idx*2]

  set txt "{$iFirst, $flags, $nRange},"
  if {$t==2} {
    puts $txt
  } else {
    puts -nonewline [format "% -23s" $txt]
  }
  set t [expr ($t+1)%3]

  return 0
}

proc tl_print_table_footer {togglevar} {
  upvar $togglevar t
  if {$t!=0} {puts ""}
  puts "  \};"
}

proc tl_print_if_entry {entry} {
  foreach {iFirst nIncr nRange nOff} $entry {}
  if {$nIncr==2} {error "tl_print_if_entry needs improvement!"}

  puts "  else if( c>=$iFirst && c<[expr $iFirst+$nRange] )\{"
  puts "    ret = c + $nOff;"
  puts "  \}"
}

proc tl_generate_ioff_table {lRecord} {
  foreach entry $lRecord {
    foreach {iFirst nIncr nRange iOff} $entry {}
    if {$iOff<0}   { incr iOff [expr (1<<16)] }
    if {[info exists a($iOff)]} continue
    set a($iOff) 1
  }

  set liOff [lsort -integer [array names a]]
  if {[llength $liOff]>128} { error "Too many distinct ioffs" }
  return $liOff
}

proc tl_print_ioff_table {liOff} {
  puts -nonewline "  static const unsigned short aiOff\[\] = \{"
  set i 0
  foreach off $liOff {
    if {($i % 8)==0} {puts "" ; puts -nonewline "   "}
    puts -nonewline [format "% -7s" "$off,"]
    incr i
  }
  puts ""
  puts "  \};"

}

proc print_fold {zFunc} {

  set lRecord [tl_create_records]

  set lHigh [list]
  puts "/*"
  puts "** Interpret the argument as a unicode codepoint. If the codepoint"
  puts "** is an upper case character that has a lower case equivalent,"
  puts "** return the codepoint corresponding to the lower case version."
  puts "** Otherwise, return a copy of the argument."
  puts "**"
  puts "** The results are undefined if the value passed to this function"
  puts "** is less than zero."
  puts "*/"
  puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{"

  set liOff [tl_generate_ioff_table $lRecord]
  tl_print_table_header
  foreach entry $lRecord { 
    if {[tl_print_table_entry toggle $entry $liOff]} { 
      lappend lHigh $entry 
    } 
  }
  tl_print_table_footer toggle
  tl_print_ioff_table $liOff

  puts [subst -nocommands {
  int ret = c;

  assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );

  if( c<128 ){
    if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
  }else if( c<65536 ){
    const struct TableEntry *p;
    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
    int iLo = 0;
    int iRes = -1;

    assert( c>aEntry[0].iCode );
    while( iHi>=iLo ){
      int iTest = (iHi + iLo) / 2;
      int cmp = (c - aEntry[iTest].iCode);
      if( cmp>=0 ){
        iRes = iTest;
        iLo = iTest+1;
      }else{
        iHi = iTest-1;
      }
    }

    assert( iRes>=0 && c>=aEntry[iRes].iCode );
    p = &aEntry[iRes];
    if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
      ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
      assert( ret>0 );
    }

    if( eRemoveDiacritic ){
      ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2);
    }
  }
  }]

  foreach entry $lHigh {
    tl_print_if_entry $entry
  }

  puts ""
  puts "  return ret;"
  puts "\}"
}

proc code {txt} {
  set txt [string trimright $txt]
  set txt [string trimleft $txt "\n"]
  set n [expr {[string length $txt] - [string length [string trim $txt]]}]
  set ret ""
  foreach L [split $txt "\n"] {
    append ret "[string range $L $n end]\n"
  }
  return [uplevel "subst -nocommands {$ret}"]
}

proc intarray {lInt} {
  set ret ""
  set n [llength $lInt]
  for {set i 0} {$i < $n} {incr i 10} {
    append ret "\n    "
    foreach int [lrange $lInt $i [expr $i+9]] {
      append ret [format "%-7s" "$int, "]
    }
  }
  append ret "\n  "
  set ret
}

proc categories_switch {Cvar first lSecond} {
  upvar $Cvar C
  set ret ""
  append ret "case '$first':\n"
  append ret "          switch( zCat\[1\] ){\n"
  foreach s $lSecond {
    append ret "            case '$s': aArray\[$C($first$s)\] = 1; break;\n"
  }
  append ret "            case '*': \n"
  foreach s $lSecond {
    append ret "              aArray\[$C($first$s)\] = 1;\n"
  }
  append ret "              break;\n"
  append ret "            default: return 1;"
  append ret "          }\n"
  append ret "          break;\n"
}

# Argument is a list. Each element of which is itself a list of two elements:
#
#   * the codepoint
#   * the category
#
# List elements are sorted in order of codepoint.
#
proc print_categories {lMap} {
  set categories {
    Cc Cf Cn Cs
    Ll Lm Lo Lt Lu
    Mc Me Mn
    Nd Nl No
    Pc Pd Pe Pf Pi Po Ps
    Sc Sk Sm So
    Zl Zp Zs

    LC Co
  }

  for {set i 0} {$i < [llength $categories]} {incr i} {
    set C([lindex $categories $i]) [expr 1+$i]
  }

  set caseC [categories_switch C C {c f n s o}]
  set caseL [categories_switch C L {l m o t u C}]
  set caseM [categories_switch C M {c e n}]
  set caseN [categories_switch C N {d l o}]
  set caseP [categories_switch C P {c d e f i o s}]
  set caseS [categories_switch C S {c k m o}]
  set caseZ [categories_switch C Z {l p s}]

  set nCat [expr [llength [array names C]] + 1]
  puts [code {
    int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
      aArray[0] = 1;
      switch( zCat[0] ){
        $caseC
        $caseL
        $caseM
        $caseN
        $caseP
        $caseS
        $caseZ
      }
      return 0;
    }
  }]

  set nRepeat 0
  set first   [lindex $lMap 0 0]
  set class   [lindex $lMap 0 1]
  set prev -1

  set CASE(0) "Lu"
  set CASE(1) "Ll"

  foreach m $lMap {
    foreach {codepoint cl} $m {}
    set codepoint [expr "0x$codepoint"]
    if {$codepoint>=(1<<20)} continue

    set bNew 0
    if {$codepoint!=($prev+1)} {
      set bNew 1
    } elseif {
      $cl==$class || ($class=="LC" && $cl==$CASE([expr $nRepeat & 0x01]))
    } {
      incr nRepeat
    } elseif {$class=="Lu" && $nRepeat==1 && $cl=="Ll"} {
      set class LC
      incr nRepeat
    } else {
      set bNew 1
    }
    if {$bNew} {
      lappend lEntries [list $first $class $nRepeat]
      set nRepeat 1
      set first $codepoint
      set class $cl
    }
    set prev $codepoint
  }
  if {$nRepeat>0} {
    lappend lEntries [list $first $class $nRepeat]
  }

  set aBlock [list 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  set aMap [list]
  foreach e $lEntries {
    foreach {cp class nRepeat} $e {}
    set block [expr ($cp>>16)]
    if {$block>0 && [lindex $aBlock $block]==0} {
      for {set i 1} {$i<=$block} {incr i} {
        if {[lindex $aBlock $i]==0} {
          lset aBlock $i [llength $aMap]
        }
      }
    }
    lappend aMap [expr {$cp & 0xFFFF}]
    lappend aData [expr {($nRepeat << 5) + $C($class)}]
  }
  for {set i 1} {$i<[llength $aBlock]} {incr i} {
    if {[lindex $aBlock $i]==0} {
      lset aBlock $i [llength $aMap]
    }
  }

  set aBlockArray [intarray $aBlock]
  set aMapArray [intarray $aMap]
  set aDataArray [intarray $aData]
  puts [code {
    static u16 aFts5UnicodeBlock[] = {$aBlockArray};
    static u16 aFts5UnicodeMap[] = {$aMapArray};
    static u16 aFts5UnicodeData[] = {$aDataArray};

    int sqlite3Fts5UnicodeCategory(u32 iCode) { 
      int iRes = -1;
      int iHi;
      int iLo;
      int ret;
      u16 iKey;

      if( iCode>=(1<<20) ){
        return 0;
      }
      iLo = aFts5UnicodeBlock[(iCode>>16)];
      iHi = aFts5UnicodeBlock[1+(iCode>>16)];
      iKey = (iCode & 0xFFFF);
      while( iHi>iLo ){
        int iTest = (iHi + iLo) / 2;
        assert( iTest>=iLo && iTest<iHi );
        if( iKey>=aFts5UnicodeMap[iTest] ){
          iRes = iTest;
          iLo = iTest+1;
        }else{
          iHi = iTest;
        }
      }

      if( iRes<0 ) return 0;
      if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
      ret = aFts5UnicodeData[iRes] & 0x1F;
      if( ret!=$C(LC) ) return ret;
      return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? $C(Ll) : $C(Lu);
    }

    void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
      int i = 0;
      int iTbl = 0;
      while( i<128 ){
        int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
        int n = (aFts5UnicodeData[iTbl] >> 5) + i;
        for(; i<128 && i<n; i++){
          aAscii[i] = (u8)bToken;
        }
        iTbl++;
      }
      aAscii[0] = 0;                  /* 0x00 is never a token character */
    }
  }]
}

proc print_test_categories {lMap} {

  set lCP [list]
  foreach e $lMap {
    foreach {cp cat} $e {}
    if {[expr 0x$cp] < (1<<20)} {
      lappend lCP "{0x$cp, \"$cat\"}, "
    }
  }

  set aCP "\n"
  for {set i 0} {$i < [llength $lCP]} {incr i 4} {
    append aCP "    [join [lrange $lCP $i $i+3]]\n"
  }


  puts [code {
    static int categories_test (int *piCode){
      struct Codepoint {
        int iCode;
        const char *zCat;
      } aCP[] = {$aCP};
      int i;
      int iCP = 0;

      for(i=0; i<1000000; i++){
        u8 aArray[40];
        int cat = 0;
        int c = 0;
        memset(aArray, 0, sizeof(aArray));
        if( aCP[iCP].iCode==i ){
          sqlite3Fts5UnicodeCatParse(aCP[iCP].zCat, aArray);
          iCP++;
        }else{
          aArray[0] = 1;
        }

        c = sqlite3Fts5UnicodeCategory((u32)i);
        if( aArray[c]==0 ){
          *piCode = i;
          return 1;
        }
      }

      return 0;
    }
  }]
}

proc print_fold_test {zFunc mappings} {
  global tl_lookup_table

  foreach m $mappings {
    set c [lindex $m 1]
    if {$c == ""} {
      set extra([lindex $m 0]) 0
    } else {
      scan $c %c i
      set extra([lindex $m 0]) $i
    }
  }

  puts "static int fold_test(int *piCode)\{"
  puts -nonewline "  static int aLookup\[\] = \{"
  for {set i 0} {$i < 70000} {incr i} {

    set expected $i
    catch { set expected $tl_lookup_table($i) }
    set expected2 $expected
    catch { set expected2 $extra($expected2) }

    if {($i % 4)==0}  { puts "" ; puts -nonewline "    " }
    puts -nonewline "$expected, $expected2, "
  }
  puts "  \};"
  puts "  int i;"
  puts "  for(i=0; i<sizeof(aLookup)/sizeof(aLookup\[0\]); i++)\{"
  puts "    int iCode = (i/2);"
  puts "    int bFlag = i & 0x0001;"
  puts "    if( ${zFunc}\(iCode, bFlag)!=aLookup\[i\] )\{"
  puts "      *piCode = iCode;"
  puts "      return 1;"
  puts "    \}"
  puts "  \}"
  puts "  return 0;"
  puts "\}"
}


proc print_fileheader {} {
  puts [string trim {
/*
** 2012-05-25
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/

/*
** DO NOT EDIT THIS MACHINE GENERATED FILE.
*/
  }]
  puts ""
  if {$::generate_fts5_code} {
    # no-op
  } else {
    puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE"
    puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)"
  }
  puts ""
  puts "#include <assert.h>"
  puts ""
}

proc print_test_main {} {
  puts ""
  puts "#include <stdio.h>"
  puts ""
  puts "int main(int argc, char **argv)\{"
  puts "  int r1, r2, r3;"
  puts "  int code;"
  puts "  r3 = 0;"
  puts "  r1 = isalnum_test(&code);"
  puts "  if( r1 ) printf(\"isalnum(): Problem with code %d\\n\",code);"
  puts "  else printf(\"isalnum(): test passed\\n\");"
  puts "  r2 = fold_test(&code);"
  puts "  if( r2 ) printf(\"fold(): Problem with code %d\\n\",code);"
  puts "  else printf(\"fold(): test passed\\n\");"
  if {$::generate_fts5_code} {
    puts "  r3 = categories_test(&code);"
    puts "  if( r3 ) printf(\"categories(): Problem with code %d\\n\",code);"
    puts "  else printf(\"categories(): test passed\\n\");"
  }
  puts "  return (r1 || r2 || r3);"
  puts "\}"
}

# Proces the command line arguments. Exit early if they are not to
# our liking.
#
proc usage {} {
  puts -nonewline stderr "Usage: $::argv0 ?-test? ?-fts5? "
  puts            stderr "<CaseFolding.txt file> <UnicodeData.txt file>"
  exit 1
}
if {[llength $argv]<2} usage
set unicodedata.txt [lindex $argv end]
set casefolding.txt [lindex $argv end-1]

set remove_diacritic remove_diacritic
set generate_test_code 0
set generate_fts5_code 0
set function_prefix "sqlite3Fts"
for {set i 0} {$i < [llength $argv]-2} {incr i} {
  switch -- [lindex $argv $i] {
    -test {
      set generate_test_code 1
    }
    -fts5 {
      set function_prefix sqlite3Fts5
      set generate_fts5_code 1
      set remove_diacritic fts5_remove_diacritic
    }
    default {
      usage
    }
  }
}

print_fileheader

if {$::generate_test_code} {
  puts "typedef unsigned short int u16;"
  puts "typedef unsigned char u8;"
  puts "#include <string.h>"
}

# Print the isalnum() function to stdout.
#
set lRange [an_load_separator_ranges]
if {$generate_fts5_code==0} {
  print_isalnum ${function_prefix}UnicodeIsalnum $lRange
}

# Leave a gap between the two generated C functions.
#
puts ""
puts ""

# Load the fold data. This is used by the [rd_XXX] commands
# as well as [print_fold].
tl_load_casefolding_txt ${casefolding.txt}

set mappings [rd_load_unicodedata_text ${unicodedata.txt}]
print_rd $mappings
puts ""
puts ""
print_isdiacritic ${function_prefix}UnicodeIsdiacritic $mappings
puts ""
puts ""

# Print the fold() function to stdout.
#
print_fold ${function_prefix}UnicodeFold

if {$generate_fts5_code} {
  puts ""
  puts ""
  print_categories [cc_load_unicodedata_text ${unicodedata.txt}]
}

# Print the test routines and main() function to stdout, if -test 
# was specified.
#
if {$::generate_test_code} {
  if {$generate_fts5_code==0} {
    print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
  }
  print_fold_test ${function_prefix}UnicodeFold $mappings
  print_test_categories [cc_load_unicodedata_text ${unicodedata.txt}]
  print_test_main 
}

if {$generate_fts5_code} {
  # no-op
} else {
  puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
  puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
}