Skip to content

Commit c52795d

Browse files
feodorfeodor
committed
Text parser rewritten:
 - supports multibyte encodings - more strict rules for lexemes - flex isn't used Add: - tsquery plainto_tsquery(text) Function makes tsquery from plain text. - &&, ||, !! operation for tsquery for combining tsquery from it's parts: 'foo & bar' || 'asd' => 'foo & bar | asd'
1 parent b91e6ed commit c52795d

File tree

15 files changed

+1613
-424
lines changed

15 files changed

+1613
-424
lines changed

‎contrib/tsearch2/Makefile‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
1-
# $PostgreSQL: pgsql/contrib/tsearch2/Makefile,v 1.11 2005/11/08 17:08:46 teodor Exp $
1+
# $PostgreSQL: pgsql/contrib/tsearch2/Makefile,v 1.12 2005/11/21 12:27:57 teodor Exp $
22

33
MODULE_big = tsearch2
44
OBJS = dict_ex.o dict.o snmap.o stopword.o common.o prs_dcfg.o \
55
dict_snowball.o dict_ispell.o dict_syn.o \
66
wparser.o wparser_def.o \
77
ts_cfg.o tsvector.o query_cleanup.o crc32.o query.o gistidx.o \
88
tsvector_op.o rank.o ts_stat.o \
9-
query_util.o query_support.o query_rewrite.o query_gist.o
9+
query_util.o query_support.o query_rewrite.o query_gist.o \
10+
ts_locale.o
1011

1112
SUBDIRS := snowball ispell wordparser
1213
SUBDIROBJS := $(SUBDIRS:%=%/SUBSYS.o)

‎contrib/tsearch2/expected/tsearch2.out‎

Lines changed: 73 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,12 @@ psql:tsearch2.sql:342: NOTICE: argument type tsvector is only a shell
1313
psql:tsearch2.sql:396: NOTICE: type "tsquery" is not yet defined
1414
DETAIL: Creating a shell type definition.
1515
psql:tsearch2.sql:401: NOTICE: argument type tsquery is only a shell
16-
psql:tsearch2.sql:544: NOTICE: type "gtsvector" is not yet defined
16+
psql:tsearch2.sql:559: NOTICE: type "gtsvector" is not yet defined
1717
DETAIL: Creating a shell type definition.
18-
psql:tsearch2.sql:549: NOTICE: argument type gtsvector is only a shell
19-
psql:tsearch2.sql:998: NOTICE: type "gtsq" is not yet defined
18+
psql:tsearch2.sql:564: NOTICE: argument type gtsvector is only a shell
19+
psql:tsearch2.sql:1054: NOTICE: type "gtsq" is not yet defined
2020
DETAIL: Creating a shell type definition.
21-
psql:tsearch2.sql:1003: NOTICE: argument type gtsq is only a shell
21+
psql:tsearch2.sql:1059: NOTICE: argument type gtsq is only a shell
2222
--tsvector
2323
SELECT '1'::tsvector;
2424
tsvector
@@ -653,7 +653,7 @@ select * from token_type('default');
653653
11 | lpart_hword | Latin part of hyphenated word
654654
12 | blank | Space symbols
655655
13 | tag | HTML Tag
656-
14 | http | HTTP head
656+
14 | protocol | Protocol head
657657
15 | hword | Hyphenated word
658658
16 | lhword | Latin hyphenated word
659659
17 | nlhword | Non-latin hyphenated word
@@ -672,14 +672,13 @@ select * from parse('default', '345 qwe@efd.r '' http://www.com/ http://aew.werc
672672
-------+--------------------------------------
673673
22 | 345
674674
12 |
675-
4 | qwe@efd.r
676-
12 |
677-
12 | '
678-
12 |
675+
1 | qwe
676+
12 | @
677+
19 | efd.r
678+
12 | '
679679
14 | http://
680680
6 | www.com
681-
12 | /
682-
12 |
681+
12 | /
683682
14 | http://
684683
5 | aew.werc.ewr/?ad=qwe&dw
685684
6 | aew.werc.ewr
@@ -700,23 +699,21 @@ select * from parse('default', '345 qwe@efd.r '' http://www.com/ http://aew.werc
700699
6 | 4aew.werc.ewr
701700
12 |
702701
14 | http://
703-
5 | 5aew.werc.ewr:8100/?
704-
6 | 5aew.werc.ewr
705-
18 | :8100/?
706-
12 |
702+
6 | 5aew.werc.ewr:8100
703+
12 | /?
707704
1 | ad
708705
12 | =
709706
1 | qwe
710707
12 | &
711708
1 | dw
712709
12 |
713710
5 | 6aew.werc.ewr:8100/?ad=qwe&dw
714-
6 | 6aew.werc.ewr
715-
18 | :8100/?ad=qwe&dw
711+
6 | 6aew.werc.ewr:8100
712+
18 | /?ad=qwe&dw
716713
12 |
717714
5 | 7aew.werc.ewr:8100/?ad=qwe&dw=%20%32
718-
6 | 7aew.werc.ewr
719-
18 | :8100/?ad=qwe&dw=%20%32
715+
6 | 7aew.werc.ewr:8100
716+
18 | /?ad=qwe&dw=%20%32
720717
12 |
721718
7 | +4.0e-10
722719
12 |
@@ -747,11 +744,15 @@ select * from parse('default', '345 qwe@efd.r '' http://www.com/ http://aew.werc
747744
1 | jf
748745
12 |
749746
1 | sdjk
750-
13 | <we hjwer <werrwe>
747+
12 | <
748+
1 | we
751749
12 |
752-
3 | ewr1
753-
12 | >
750+
1 | hjwer
751+
12 |
752+
13 | <werrwe>
754753
12 |
754+
3 | ewr1
755+
12 | >
755756
3 | ewri2
756757
12 |
757758
13 | <a href="qwe<qwe>">
@@ -767,57 +768,53 @@ select * from parse('default', '345 qwe@efd.r '' http://www.com/ http://aew.werc
767768
12 |
768769
19 | /wqe-324/ewr
769770
12 |
770-
6 | gist.h
771-
12 |
772-
6 | gist.h.c
771+
19 | gist.h
773772
12 |
774-
6 | gist.c
775-
12 | .
773+
19 | gist.h.c
776774
12 |
775+
19 | gist.c
776+
12 | .
777777
1 | readline
778778
12 |
779779
20 | 4.2
780780
12 |
781781
20 | 4.2
782-
12 | .
783-
12 |
782+
12 | .
784783
20 | 4.2
785-
12 | ,
786-
12 |
787-
15 | readline-4
784+
12 | ,
785+
15 | readline-4.2
788786
11 | readline
789787
12 | -
790788
20 | 4.2
791789
12 |
792-
15 | readline-4
790+
15 | readline-4.2
793791
11 | readline
794792
12 | -
795793
20 | 4.2
796-
12 | .
797-
12 |
794+
12 | .
798795
22 | 234
799796
12 |
800797

801-
13 | <i <b>
798+
12 | <
799+
1 | i
800+
12 |
801+
13 | <b>
802802
12 |
803803
1 | wow
804804
12 |
805-
12 | <
806-
12 |
805+
12 | <
807806
1 | jqw
808807
12 |
809-
12 | <
810-
12 | >
811-
12 |
808+
12 | <>
812809
1 | qwerty
813-
(138 rows)
810+
(135 rows)
814811

815812
SELECT to_tsvector('default', '345 qwe@efd.r '' http://www.com/ http://aew.werc.ewr/?ad=qwe&dw 1aew.werc.ewr/?ad=qwe&dw 2aew.werc.ewr http://3aew.werc.ewr/?ad=qwe&dw http://4aew.werc.ewr http://5aew.werc.ewr:8100/? ad=qwe&dw 6aew.werc.ewr:8100/?ad=qwe&dw 7aew.werc.ewr:8100/?ad=qwe&dw=%20%32 +4.0e-10 qwe qwe qwqwe 234.435 455 5.005 teodor@stack.net qwe-wer asdf <fr>qwer jf sdjk<we hjwer <werrwe> ewr1> ewri2 <a href="qwe<qwe>">
816813
/usr/local/fff /awdf/dwqe/4325 rewt/ewr wefjn /wqe-324/ewr gist.h gist.h.c gist.c. readline 4.2 4.2. 4.2, readline-4.2 readline-4.2. 234
817814
<i <b> wow < jqw <> qwerty');
818-
to_tsvector
819-
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
820-
'ad':18 'dw':20 'jf':40 '234':62 '345':1 '4.2':53,54,55,58,61 '455':32 'jqw':64 'qwe':19,28,29,36 'wer':37 'wow':63 'asdf':38 'ewr1':42 'qwer':39 'sdjk':41 '5.005':33 'ewri2':43 'qwqwe':30 'wefjn':47 'gist.c':51 'gist.h':49 'qwerti':65 '234.435':31 ':8100/?':17 'qwe-wer':35 'readlin':52,57,60 'www.com':3 '+4.0e-10':27 'gist.h.c':50 'rewt/ewr':46 'qwe@efd.r':2 'readline-4':56,59 '/?ad=qwe&dw':6,9,13 '/wqe-324/ewr':48 'aew.werc.ewr':5 '1aew.werc.ewr':8 '2aew.werc.ewr':10 '3aew.werc.ewr':12 '4aew.werc.ewr':14 '5aew.werc.ewr':16 '6aew.werc.ewr':22 '7aew.werc.ewr':25 '/usr/local/fff':44 '/awdf/dwqe/4325':45 ':8100/?ad=qwe&dw':23 'teodor@stack.net':34 '5aew.werc.ewr:8100/?':15 ':8100/?ad=qwe&dw=%20%32':26 'aew.werc.ewr/?ad=qwe&dw':4 '1aew.werc.ewr/?ad=qwe&dw':7 '3aew.werc.ewr/?ad=qwe&dw':11 '6aew.werc.ewr:8100/?ad=qwe&dw':21 '7aew.werc.ewr:8100/?ad=qwe&dw=%20%32':24
815+
to_tsvector
816+
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
817+
'ad':17 'dw':19 'jf':39 '234':63 '345':1 '4.2':54,55,56,59,62 '455':31 'jqw':66 'qwe':2,18,27,28,35 'wer':36 'wow':65 'asdf':37 'ewr1':43 'qwer':38 'sdjk':40 '5.005':32 'efd.r':3 'ewri2':44 'hjwer':42 'qwqwe':29 'wefjn':48 'gist.c':52 'gist.h':50 'qwerti':67 '234.435':30 'qwe-wer':34 'readlin':53,58,61 'www.com':4 '+4.0e-10':26 'gist.h.c':51 'rewt/ewr':47 '/?ad=qwe&dw':7,10,14,22 '/wqe-324/ewr':49 'aew.werc.ewr':6 'readline-4.2':57,60 '1aew.werc.ewr':9 '2aew.werc.ewr':11 '3aew.werc.ewr':13 '4aew.werc.ewr':15 '/usr/local/fff':45 '/awdf/dwqe/4325':46 'teodor@stack.net':33 '/?ad=qwe&dw=%20%32':25 '5aew.werc.ewr:8100':16 '6aew.werc.ewr:8100':21 '7aew.werc.ewr:8100':24 'aew.werc.ewr/?ad=qwe&dw':5 '1aew.werc.ewr/?ad=qwe&dw':8 '3aew.werc.ewr/?ad=qwe&dw':12 '6aew.werc.ewr:8100/?ad=qwe&dw':20 '7aew.werc.ewr:8100/?ad=qwe&dw=%20%32':23
821818
(1 row)
822819

823820
SELECT length(to_tsvector('default', '345 qw'));
@@ -831,7 +828,7 @@ SELECT length(to_tsvector('default', '345 qwe@efd.r '' http://www.com/ http://ae
831828
<i <b> wow < jqw <> qwerty'));
832829
length
833830
--------
834-
53
831+
51
835832
(1 row)
836833

837834
select to_tsquery('default', 'qwe & sKies ');
@@ -876,6 +873,36 @@ select to_tsquery('default', '(the|and&(i&1))&fghj');
876873
'1' & 'fghj'
877874
(1 row)
878875

876+
select plainto_tsquery('default', 'the and z 1))& fghj');
877+
plainto_tsquery
878+
--------------------
879+
'z' & '1' & 'fghj'
880+
(1 row)
881+
882+
select plainto_tsquery('default', 'foo bar') && plainto_tsquery('default', 'asd');
883+
?column?
884+
-----------------------
885+
'foo' & 'bar' & 'asd'
886+
(1 row)
887+
888+
select plainto_tsquery('default', 'foo bar') || plainto_tsquery('default', 'asd fg');
889+
?column?
890+
------------------------------
891+
'foo' & 'bar' | 'asd' & 'fg'
892+
(1 row)
893+
894+
select plainto_tsquery('default', 'foo bar') || !!plainto_tsquery('default', 'asd fg');
895+
?column?
896+
-----------------------------------
897+
'foo' & 'bar' | !( 'asd' & 'fg' )
898+
(1 row)
899+
900+
select plainto_tsquery('default', 'foo bar') && 'asd | fg';
901+
?column?
902+
----------------------------------
903+
'foo' & 'bar' & ( 'asd' | 'fg' )
904+
(1 row)
905+
879906
select 'a b:89 ca:23A,64b d:34c'::tsvector @@ 'd:AC & ca';
880907
?column?
881908
----------

‎contrib/tsearch2/query.c‎

Lines changed: 78 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,20 @@ Datum to_tsquery_name(PG_FUNCTION_ARGS);
5151
PG_FUNCTION_INFO_V1(to_tsquery_current);
5252
Datum to_tsquery_current(PG_FUNCTION_ARGS);
5353

54+
PG_FUNCTION_INFO_V1(plainto_tsquery);
55+
Datum plainto_tsquery(PG_FUNCTION_ARGS);
56+
57+
PG_FUNCTION_INFO_V1(plainto_tsquery_name);
58+
Datum plainto_tsquery_name(PG_FUNCTION_ARGS);
59+
60+
PG_FUNCTION_INFO_V1(plainto_tsquery_current);
61+
Datum plainto_tsquery_current(PG_FUNCTION_ARGS);
62+
5463
/* parser's states */
5564
#define WAITOPERAND 1
5665
#define WAITOPERATOR 2
5766
#define WAITFIRSTOPERAND 3
67+
#define WAITSINGLEOPERAND 4
5868

5969
/*
6070
* node of query tree, also used
@@ -195,6 +205,14 @@ gettoken_query(QPRS_STATE * state, int4 *val, int4 *lenval, char **strval, int2
195205
else if (*(state->buf) != ' ')
196206
return ERR;
197207
break;
208+
case WAITSINGLEOPERAND:
209+
if ( *(state->buf) == '\0' )
210+
return END;
211+
*strval = state->buf;
212+
*lenval = strlen( state->buf );
213+
state->buf += strlen( state->buf );
214+
state->count++;
215+
return VAL;
198216
default:
199217
return ERR;
200218
break;
@@ -582,7 +600,7 @@ findoprnd(ITEM * ptr, int4 *pos)
582600
* input
583601
*/
584602
static QUERYTYPE *
585-
queryin(char *buf, void (*pushval) (QPRS_STATE *, int, char *, int, int2), int cfg_id)
603+
queryin(char *buf, void (*pushval) (QPRS_STATE *, int, char *, int, int2), int cfg_id, bool isplain)
586604
{
587605
QPRS_STATE state;
588606
int4 i;
@@ -599,7 +617,7 @@ static QUERYTYPE *
599617

600618
/* init state */
601619
state.buf = buf;
602-
state.state = WAITFIRSTOPERAND;
620+
state.state = (isplain) ? WAITSINGLEOPERAND : WAITFIRSTOPERAND;
603621
state.count = 0;
604622
state.num = 0;
605623
state.str = NULL;
@@ -679,7 +697,7 @@ Datum
679697
tsquery_in(PG_FUNCTION_ARGS)
680698
{
681699
SET_FUNCOID();
682-
PG_RETURN_POINTER(queryin((char *) PG_GETARG_POINTER(0), pushval_asis, 0));
700+
PG_RETURN_POINTER(queryin((char *) PG_GETARG_POINTER(0), pushval_asis, 0, false));
683701
}
684702

685703
/*
@@ -910,7 +928,7 @@ to_tsquery(PG_FUNCTION_ARGS)
910928
str = text2char(in);
911929
PG_FREE_IF_COPY(in, 1);
912930

913-
query = queryin(str, pushval_morph, PG_GETARG_INT32(0));
931+
query = queryin(str, pushval_morph, PG_GETARG_INT32(0),false);
914932

915933
if ( query->size == 0 )
916934
PG_RETURN_POINTER(query);
@@ -950,3 +968,59 @@ to_tsquery_current(PG_FUNCTION_ARGS)
950968
Int32GetDatum(get_currcfg()),
951969
PG_GETARG_DATUM(0)));
952970
}
971+
972+
Datum
973+
plainto_tsquery(PG_FUNCTION_ARGS)
974+
{
975+
text *in = PG_GETARG_TEXT_P(1);
976+
char *str;
977+
QUERYTYPE *query;
978+
ITEM *res;
979+
int4 len;
980+
981+
SET_FUNCOID();
982+
983+
str = text2char(in);
984+
PG_FREE_IF_COPY(in, 1);
985+
986+
query = queryin(str, pushval_morph, PG_GETARG_INT32(0), true);
987+
988+
if ( query->size == 0 )
989+
PG_RETURN_POINTER(query);
990+
991+
res = clean_fakeval_v2(GETQUERY(query), &len);
992+
if (!res)
993+
{
994+
query->len = HDRSIZEQT;
995+
query->size = 0;
996+
PG_RETURN_POINTER(query);
997+
}
998+
memcpy((void *) GETQUERY(query), (void *) res, len * sizeof(ITEM));
999+
pfree(res);
1000+
PG_RETURN_POINTER(query);
1001+
}
1002+
1003+
Datum
1004+
plainto_tsquery_name(PG_FUNCTION_ARGS)
1005+
{
1006+
text *name = PG_GETARG_TEXT_P(0);
1007+
Datum res;
1008+
1009+
SET_FUNCOID();
1010+
res = DirectFunctionCall2(plainto_tsquery,
1011+
Int32GetDatum(name2id_cfg(name)),
1012+
PG_GETARG_DATUM(1));
1013+
1014+
PG_FREE_IF_COPY(name, 0);
1015+
PG_RETURN_DATUM(res);
1016+
}
1017+
1018+
Datum
1019+
plainto_tsquery_current(PG_FUNCTION_ARGS)
1020+
{
1021+
SET_FUNCOID();
1022+
PG_RETURN_DATUM(DirectFunctionCall2(plainto_tsquery,
1023+
Int32GetDatum(get_currcfg()),
1024+
PG_GETARG_DATUM(0)));
1025+
}
1026+

0 commit comments

Comments
 (0)