ext/fts5/test/fts5origintext.test

# 2014 Jan 08
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focused on phrase queries.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5origintext

# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

foreach_detail_mode $testprefix {
foreach {tn insttoken} {
  1 0
  2 1
} {
reset_db

sqlite3_fts5_register_origintext db
do_execsql_test $tn.1.0 {
  CREATE VIRTUAL TABLE ft USING fts5(
      x, tokenize="origintext unicode61", detail=%DETAIL%
  );
  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
}

do_execsql_test $tn.1.1 {
  INSERT INTO ft VALUES('Hello world');
}

do_execsql_test $tn.1.2 {
  INSERT INTO ft(ft) VALUES('integrity-check');
}

proc b {x} { string map [list "\0" "."] $x }
db func b b

do_execsql_test $tn.1.3 {
  select b(term) from vocab;
} {
  hello.Hello
  world
}

do_execsql_test $tn.1.4 {
  SELECT rowid FROM ft('Hello');
} {1}

#-------------------------------------------------------------------------
reset_db

# Return a random integer between 0 and n-1.
#
proc random {n} {
  expr {abs(int(rand()*$n))}
}

proc select_one {list} {
  set n [llength $list]
  lindex $list [random $n]
}

proc term {} {
  set first_letter {
    a b c d e f g h i j k l m n o p q r s t u v w x y z
    A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
  }

  set term [select_one $first_letter]
  append term [random 100]
}

proc document {} {
  set nTerm [expr [random 5] + 5]
  set doc ""
  for {set ii 0} {$ii < $nTerm} {incr ii} {
    lappend doc [term]
  }
  set doc
}
db func document document

sqlite3_fts5_register_origintext db
do_execsql_test $tn.2.0 {
  CREATE VIRTUAL TABLE ft USING fts5(
      x, tokenize="origintext unicode61", detail=%DETAIL%
  );
  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
  INSERT INTO ft(ft, rank) VALUES('pgsz', 128);
  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
}

do_test $tn.2.1 {
  for {set ii 0} {$ii < 500} {incr ii} {
    execsql { INSERT INTO ft VALUES( document() ) }
  }
} {}

do_execsql_test $tn.2.2 {
  INSERT INTO ft(ft) VALUES('integrity-check');
}

do_execsql_test $tn.2.3 {
  INSERT INTO ft(ft, rank) VALUES('merge', 16);
}

do_execsql_test $tn.2.4 {
  INSERT INTO ft(ft) VALUES('integrity-check');
}

do_execsql_test $tn.2.5 {
  INSERT INTO ft(ft) VALUES('optimize');
}

#-------------------------------------------------------------------------
reset_db

sqlite3_fts5_register_origintext db
do_execsql_test $tn.3.0 {
  CREATE VIRTUAL TABLE ft USING fts5(
      x, tokenize="origintext unicode61", detail=%DETAIL%
  );
  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);

  INSERT INTO ft(rowid, x) VALUES(1, 'hello');
  INSERT INTO ft(rowid, x) VALUES(2, 'Hello');
  INSERT INTO ft(rowid, x) VALUES(3, 'HELLO');
}

#proc b {x} { string map [list "\0" "."] $x }
#db func b b
#execsql_pp { SELECT b(term) FROM vocab }

do_execsql_test $tn.3.1.1 { SELECT rowid FROM ft('hello') } 1
do_execsql_test $tn.3.1.2 { SELECT rowid FROM ft('Hello') } 2
do_execsql_test $tn.3.1.3 { SELECT rowid FROM ft('HELLO') } 3

do_execsql_test $tn.3.2 {
  CREATE VIRTUAL TABLE ft2 USING fts5(x, 
      tokenize="origintext unicode61", 
      tokendata=1,
      detail=%DETAIL%
  );
  INSERT INTO ft2(ft2, rank) VALUES('insttoken', $insttoken);
  CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance);

  INSERT INTO ft2(rowid, x) VALUES(1, 'hello');
  INSERT INTO ft2(rowid, x) VALUES(2, 'Hello');
  INSERT INTO ft2(rowid, x) VALUES(3, 'HELLO');

  INSERT INTO ft2(rowid, x) VALUES(10, 'helloooo');
}

#proc b {x} { string map [list "\0" "."] $x }
#db func b b
#execsql_pp { SELECT b(term) FROM vocab }

do_execsql_test $tn.3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3}
do_execsql_test $tn.3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3}
do_execsql_test $tn.3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3}

do_execsql_test $tn.3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10}

do_execsql_test $tn.3.3.5.1 { SELECT rowid FROM ft2('HELLO') ORDER BY rowid DESC} {
  3 2 1
}
do_execsql_test $tn.3.3.5.2 { SELECT rowid FROM ft2('HELLO') ORDER BY +rowid DESC} {
  3 2 1
}

#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
proc querytoken {cmd iPhrase iToken} { 
  set txt [$cmd xQueryToken $iPhrase $iToken]
  string map [list "\0" "."] $txt
}
sqlite3_fts5_create_function db querytoken querytoken

do_execsql_test $tn.4.0 {
  CREATE VIRTUAL TABLE ft USING fts5(
      x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
  );
  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
  INSERT INTO ft VALUES('one two three four');
}

do_execsql_test $tn.4.1 {
  SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO')
} {1 two.TwO}
do_execsql_test $tn.4.2 {
  SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE')
} {1 one}
do_execsql_test $tn.4.3 {
  SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE')
} {1 two.TWO}

if {"%DETAIL%"=="full"} {
  # Phrase queries are only supported for detail=full.
  #
  do_execsql_test $tn.4.4 {
    SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"')
  } {1 three.ThreE}
  do_catchsql_test $tn.4.5 {
    SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"')
  } {1 SQLITE_RANGE}
  do_catchsql_test $tn.4.6 {
    SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"')
  } {1 SQLITE_RANGE}
  do_catchsql_test $tn.4.7 {
    SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"')
  } {1 SQLITE_RANGE}
}

#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
proc insttoken {cmd iIdx iToken} { 
  set txt [$cmd xInstToken $iIdx $iToken]
  string map [list "\0" "."] $txt
}
sqlite3_fts5_create_function db insttoken insttoken
fts5_aux_test_functions db

do_execsql_test $tn.5.0 {
  CREATE VIRTUAL TABLE ft USING fts5(
      x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
  );
  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
  INSERT INTO ft VALUES('one ONE One oNe oNE one');
}

do_execsql_test $tn.5.1 {
  SELECT insttoken(ft, 0, 0), 
         insttoken(ft, 1, 0),
         insttoken(ft, 2, 0),
         insttoken(ft, 3, 0),
         insttoken(ft, 4, 0),
         insttoken(ft, 5, 0)
  FROM ft('one');
} {
  one one.ONE one.One one.oNe one.oNE one
}

do_execsql_test $tn.5.2 {
  SELECT insttoken(ft, 0, 0), 
         insttoken(ft, 1, 0),
         insttoken(ft, 2, 0),
         insttoken(ft, 3, 0),
         insttoken(ft, 4, 0),
         insttoken(ft, 5, 0)
  FROM ft('on*');
} {
  one one.ONE one.One one.oNe one.oNE one
}

do_execsql_test $tn.5.3 {
  SELECT insttoken(ft, 0, 0), 
         insttoken(ft, 1, 0),
         insttoken(ft, 2, 0),
         insttoken(ft, 3, 0),
         insttoken(ft, 4, 0),
         insttoken(ft, 5, 0)
  FROM ft(fts5_insttoken('on*'));
} {
  one one.ONE one.One one.oNe one.oNE one
}

do_execsql_test $tn.5.4 {
  SELECT insttoken(ft, 1, 0) FROM ft('one');
} {
  one.ONE
}

do_execsql_test $tn.5.5 {
  SELECT fts5_test_poslist(ft) FROM ft('one');
} {
  {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5}
}

#-------------------------------------------------------------------------
# Test the xInstToken() API with:
#
#   * a non tokendata=1 table.
#   * prefix queries.
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test $tn.6.0 {
  CREATE VIRTUAL TABLE ft USING fts5( 
      x, y, tokenize='origintext unicode61', detail=%DETAIL%, tokendata=0
  );
  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);

  INSERT INTO ft VALUES('One Two', 'Three two');
  INSERT INTO ft VALUES('three Three', 'one One');
}
proc tokens {cmd} { 
  set ret [list]
  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
    set txt [$cmd xInstToken $iTok 0]
    set txt [string map [list "\0" "."] $txt]
    lappend ret $txt
  }
  set ret
}
sqlite3_fts5_create_function db tokens tokens

do_execsql_test $tn.6.1 {
  SELECT rowid, tokens(ft) FROM ft('One');
} {1 one.One 2 one.One}

do_execsql_test $tn.6.2 {
  SELECT rowid, tokens(ft) FROM ft('on*');
} {1 one.One 2 {one one.One}}

do_execsql_test $tn.6.3 {
  SELECT rowid, tokens(ft) FROM ft('Three*');
} {1 three.Three 2 three.Three}

fts5_aux_test_functions db
do_catchsql_test $tn.6.4 {
  SELECT fts5_test_insttoken(ft, -1, 0) FROM ft('one');
} {1 SQLITE_RANGE}

do_catchsql_test $tn.6.5 {
  SELECT fts5_test_insttoken(ft, 1, 0) FROM ft('one');
} {1 SQLITE_RANGE}

do_catchsql_test $tn.6.6 {
  CREATE VIRTUAL TABLE ft2 USING fts5(x, tokendata=2);
} {1 {malformed tokendata=... directive}}
do_catchsql_test $tn.6.7 {
  CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', tokendata=11);
} {1 {malformed tokendata=... directive}}

}
}

finish_test