Please start any new threads on our new site at https://forums.sqlteam.com. We've got lots of great SQL Server experts to answer whatever question you can come up with.

 All Forums
 SQL Server 2000 Forums
 SQL Server Development (2000)
 Eliminate certain words in keywords in stored proc

Author  Topic 

thiruna
Starting Member

41 Posts

Posted - 2002-08-01 : 04:53:50
Eliminating certain words from the keyword using stored procedure
Hi,
I have developed the following stored procedure for searching the table by name lessons

CREATE PROCEDURE SP_search_lessons_new
(
@page int =1,
@recsperpage int,
@search_division int =null,
@search_project int =null,
@search_country int =null,
@search_phase int =null,
@search_department int =null,
@search_date datetime =null,
@search_lesson_approved int =null,
@search_lesson_complete int =null,
@search_keyword varchar(200) =null
)
AS
-- We don't want to return the # of rows inserted
-- into our temporary table, so turn NOCOUNT ON
SET NOCOUNT ON
--Create a temporary table

IF @search_keyword is NOT NULL

BEGIN
SET @search_keyword =LTRIM(RTRIM(@search_keyword))
IF @search_keyword =''
BEGIN
SET @search_keyword =null
END
END

CREATE TABLE #TempItems
(
ID int IDENTITY,
lesson_no int,
division_id varchar(10),
division_name varchar(50),
project_id varchar(10),
project_short_name varchar(50),
project_name varchar(150),
country_id varchar(10),
country_name varchar(50),
phase_id varchar(10),
phase_name varchar(50),
department_id varchar(10),
department_name varchar(50),
date_time datetime,
approved bit,
complete bit,
lesson_title varchar(200)
)


-- Insert the rows from tblItems into the temp. table
INSERT INTO #TempItems (lesson_no, division_id, division_name, project_id, project_short_name, project_name, country_id, country_name, phase_id, phase_name, department_id, department_name, date_time, approved, complete, lesson_title)
SELECT lesson_no, division_id, division_name, project_id, project_short_name, project_name, country_id, country_name, phase_id, phase_name, department_id, department_name, date_time, approved, complete, lesson_title
FROM view_lessons_admin
WHERE division_no = ISNULL(@search_division, division_no)
AND project_no = ISNULL(@search_project, project_no)
AND country_no = ISNULL(@search_country, country_no)
AND phase_no = ISNULL(@search_phase, phase_no)
AND department_no = ISNULL(@search_department, department_no)
AND date_time >= ISNULL(@search_date, date_time)
AND approved = ISNULL(@search_lesson_approved, approved)
AND complete = ISNULL(@search_lesson_complete, complete)
AND (lesson_description LIKE ISNULL('%' + @search_keyword + '%', lesson_description)
OR lesson_title LIKE ISNULL( '%' + @search_keyword + '%', lesson_title)
OR actions_taken LIKE ISNULL('%' + @search_keyword + '%', actions_taken)
OR lesson_learnt LIKE ISNULL('%' + @search_keyword + '%' , lesson_learnt))
--ORDER BY date_time DESC

IF @search_keyword is NOT NULL
--here i am parsing the keywords(thanks to graz) and searching for each word in the table
--i want to eliminate the some words say 'a', 'is', etc..
--how would i do it in SQL...??

BEGIN
declare @separator_position int -- Locates each separator character
declare @array_value varchar(1000) -- Holds each returned array value
declare @like_text varchar (1000)
declare @separator char(1)
declare @keyword_count int
set @separator =' '

set @search_keyword = @search_keyword + @separator
set @keyword_count = 0

while patindex('%' + @separator + '%' , @search_keyword) <> 0
begin
select @separator_position = patindex('%' + @separator + '%' , @search_keyword)
select @array_value = left(@search_keyword, @separator_position - 1)
select @like_text = '%' + @array_value + '%'

INSERT INTO #TempItems (lesson_no, division_id, division_name, project_id, project_short_name, project_name, country_id, country_name, phase_id, phase_name, department_id, department_name, date_time, approved, complete, lesson_title)
SELECT lesson_no, division_id, division_name, project_id, project_short_name, project_name, country_id, country_name, phase_id, phase_name, department_id, department_name, date_time, approved, complete, lesson_title
FROM view_lessons_admin
WHERE division_no = ISNULL(@search_division, division_no)
AND project_no = ISNULL(@search_project, project_no)
AND country_no = ISNULL(@search_country, country_no)
AND phase_no = ISNULL(@search_phase, phase_no)
AND department_no = ISNULL(@search_department, department_no)
AND date_time >= ISNULL(@search_date, date_time)
AND approved = ISNULL(@search_lesson_approved, approved)
AND complete = ISNULL(@search_lesson_complete, complete)
AND (lesson_description LIKE ISNULL(@like_text , lesson_description)
OR lesson_title LIKE ISNULL(@like_text, lesson_title)
OR actions_taken LIKE ISNULL(@like_text, actions_taken)
OR lesson_learnt LIKE ISNULL(@like_text , lesson_learnt))
--ORDER BY date_time DESC
select @search_keyword = stuff(@search_keyword, 1, @separator_position, '')
set @keyword_count =@keyword_count+1
end

end

IF @keyword_count =1
BEGIN
--to correct the ratings at the search page to not go above 100
SET @keyword_count =2
END

-- Find out the first and last record we want
DECLARE @FirstRec int, @LastRec int
SELECT @FirstRec = (@Page - 1) * @RecsPerPage
SELECT @LastRec = (@Page * @RecsPerPage + 1)
-- Now, return the set of paged records, plus, an indiciation of we
-- have more records or not!
SELECT distinct lesson_no, Rating =count(*), division_id, division_name, project_id, project_short_name, project_name, country_id, country_name, phase_id, phase_name, department_id, department_name, date_time, approved, complete, LTRIM(LEFT(lesson_title, 30))+'...' AS lesson_title ,
MoreRecords =
(
SELECT COUNT( distinct lesson_no)
FROM #TempItems TI
WHERE TI.ID >= @LastRec
) ,
Totalrecords =
(
SELECT COUNT( distinct lesson_no)
FROM #TempItems TI
),
keywordcount =
(
SELECT @keyword_count
)
FROM #TempItems
WHERE ID > @FirstRec AND ID < @LastRec
Group by lesson_no, division_id, division_name, project_id, project_short_name, project_name, country_id, country_name, phase_id, phase_name, department_id, department_name, date_time, approved, complete, lesson_title
ORDER BY 2 DESC, date_time DESC
-- Turn NOCOUNT back OFF
SET NOCOUNT OFF
GO

Also advise whether it is a viable method or any other efficient method there to search...?



Edited by - thiruna on 08/01/2002 08:39:21

Onamuji
Aged Yak Warrior

504 Posts

Posted - 2002-08-01 : 08:18:17
just a thought....

set @search_keyword = replace(' ' + @search_keyword + ' ', ' a ', ' ')
set @search_keyword = replace(' ' + @search_keyword + ' ', ' is ', ' ')

though you would need to do this for every word




Edited by - onamuji on 08/01/2002 08:18:33
Go to Top of Page

dsdeming

479 Posts

Posted - 2002-08-01 : 08:20:33
Couldn't you use the REPLACE() function:

SET @string = REPLACE( @string, ' a ', ' ' )
SET @string = REPLACE( @string, ' is ', ' ' )

etc.

@$#@$%^% Sniped again!

Edited by - dsdeming on 08/01/2002 08:21:30
Go to Top of Page

thiruna
Starting Member

41 Posts

Posted - 2002-08-01 : 08:45:50
thanks for that....
but this is not the one i am looking for...

I want to know whether can we put the values to be eluminated, in a array and check for values in the array everytime before proceeding to search.

or can we use table and feed in the words that we dont want to search and so we can check for the words in the table and if exists, it will skip that word and goes to the next one....

Which one do you think is viable....?

many thanks
thiru

Go to Top of Page

Page47
Master Smack Fu Yak Hacker

2878 Posts

Posted - 2002-08-01 : 09:37:30
It would be alot easier if, before you search, you put your parsed keywords into a table and then eliminated the bad ones. Then join your view_lessons_admin to the table of keywords to get your rowset.

Jay White
{0}
Go to Top of Page

thiruna
Starting Member

41 Posts

Posted - 2002-08-01 : 11:09:07
jay,

could u please show it in programming...i didnt understand ur concept please...

thanks
thiru


Go to Top of Page

Page47
Master Smack Fu Yak Hacker

2878 Posts

Posted - 2002-08-01 : 11:32:21
I am suggestion you modify Graz's code, such that rather returning a recordset for each element in your csv, it returns on complete rowset....

create procedure usp_ParseArray
( @Array varchar(1000),
@separator char(1) )
AS
-- Created by graz@sqlteam.com
set nocount on
-- @Array is the array we wish to parse
-- @Separator is the separator charactor such as a comma
declare @separator_position int -- This is used to locate each separator character
declare @array_value varchar(1000) -- this holds each array value as it is returned
create table #array (value varchar(1000) not null)

-- For my loop to work I need an extra separator at the end. I always look to the
-- left of the separator character for each array value
set @array = @array + @separator

-- Loop through the string searching for separtor characters
while patindex('%' + @separator + '%' , @array) <> 0
begin

-- patindex matches the a pattern against a string
select @separator_position = patindex('%' + @separator + '%' , @array)
select @array_value = left(@array, @separator_position - 1)

-- This is where you process the values passed.
-- Replace this select statement with your processing
-- @array_value holds the value of this element of the array
insert into #array
select @array_value

-- This replaces what we just processed with and empty string
select @array = stuff(@array, 1, @separator_position, '')
end

select * from #array
set nocount off
go

 
Now, you can pass your @search_keyword variable to this stored procedure to break it up into its components....You'll want to capture this in a temp table...

create table #search (value varchar(1000))
insert into #search
exec usp_parsearray @search_keyword, ','

 
For here its simple a matter of removing the ones you don't want...

delete #search
from #search s
inner join reservedwordtable rwt
on s.value = rwt.word

 
Now you have a table with you search words, minus the throw aways.

You'll need to modify your where clause to look in the table rather than doing a LIKE on the variable, but that shouldnt be too hard....




Jay White
{0}

Edited by - Page47 on 08/01/2002 11:33:12
Go to Top of Page

thiruna
Starting Member

41 Posts

Posted - 2002-08-02 : 00:49:50
jay,

i have followed as per your message....and remodified as follows...

CREATE PROCEDURE SP_search_lessons_new_filter_keywords
(
...as above
)
cont...

IF @search_keyword is NOT NULL

BEGIN
CREATE TABLE #search
(
value varchar(1000)
)

INSERT INTO #search
EXEC sp_parsearray @search_keyword, ' '

--created a table by name reservedwords and inserted some words that have to be excluded...
DELETE #search
FROM #search s
INNER JOIN reservedwords rwrd
ON s.value = rwrd.word

INSERT INTO #TempItems (lesson_no, division_id, division_name, project_id, project_short_name, project_name, country_id, country_name, phase_id, phase_name, department_id, department_name, date_time, approved, complete, lesson_title)
SELECT lesson_no, division_id, division_name, project_id, project_short_name, project_name, country_id, country_name, phase_id, phase_name, department_id, department_name, date_time, approved, complete, lesson_title
FROM view_lessons_admin
WHERE division_no = ISNULL(@search_division, division_no)
AND project_no = ISNULL(@search_project, project_no)
AND country_no = ISNULL(@search_country, country_no)
AND phase_no = ISNULL(@search_phase, phase_no)
AND department_no = ISNULL(@search_department, department_no)
AND date_time >= ISNULL(@search_date, date_time)
AND approved = ISNULL(@search_lesson_approved, approved)
AND complete = ISNULL(@search_lesson_complete, complete)
AND lesson_description IN (SELECT * FROM #search)
OR lesson_title IN (SELECT * FROM #search)
OR actions_taken IN (SELECT * FROM #search)
OR lesson_learnt IN (SELECT * FROM #search)


SELECT @keyword_count =count(*) from #search

END

IF @keyword_count =1
BEGIN
--to correct the ratings at the search page to not go above 100
SET @keyword_count =2
END

-- Find out the first and last record we want
DECLARE @FirstRec int, @LastRec int
SELECT @FirstRec = (@Page - 1) * @RecsPerPage
SELECT @LastRec = (@Page * @RecsPerPage + 1)
-- Now, return the set of paged records, plus, an indiciation of we
-- have more records or not!
SELECT distinct lesson_no, Rating =count(*), division_id, division_name, project_id, project_short_name, project_name, country_id, country_name, phase_id, phase_name, department_id, department_name, date_time, approved, complete, LTRIM(LEFT(lesson_title, 30))+'...' AS lesson_title ,
MoreRecords =
(
SELECT COUNT( distinct lesson_no)
FROM #TempItems TI
WHERE TI.ID >= @LastRec
) ,
Totalrecords =
(
SELECT COUNT( distinct lesson_no)
FROM #TempItems TI
),
keywordcount =
(
SELECT @keyword_count
)
FROM #TempItems
WHERE ID > @FirstRec AND ID < @LastRec
Group by lesson_no, division_id, division_name, project_id, project_short_name, project_name, country_id, country_name, phase_id, phase_name, department_id, department_name, date_time, approved, complete, lesson_title
ORDER BY 2 DESC, date_time DESC
-- Turn NOCOUNT back OFF
SET NOCOUNT OFF
GO

I am getting the error from SQL saying "The text, ntext, and image data types cannot be compared or sorted, except when using IS NULL or LIKE operator. "

of course, the field types which i am using are text fields..


is there any trick to overcome this problem...?

many thanks
thiru


Edited by - thiruna on 08/02/2002 06:59:03

Edited by - thiruna on 08/02/2002 07:02:20

Edited by - thiruna on 08/02/2002 07:08:40
Go to Top of Page

Page47
Master Smack Fu Yak Hacker

2878 Posts

Posted - 2002-08-02 : 08:22:24
ok, then go back to using your LIKE . . .

After you create and clean up the #search table, form a single string from it...

declare @search_keyword_clean varchar(8000)
select
@search_keyword_clean = coalesce(@search_keyword_clean + ' ', '') + value
from
#search

 


Jay White
{0}
Go to Top of Page

thiruna
Starting Member

41 Posts

Posted - 2002-08-03 : 01:27:16
jay,

the one which you have shown me will merge the words back, after removing the unwanted words...

if we use 'Like' operator for the whole merged keyword, the probability of finding the records will be less when compared to search for each and every word in the table....

is there any option in SQL so that even if we supply the set of words together, SQL will search for every word in the table...

many thanks
thiru


Go to Top of Page

thiruna
Starting Member

41 Posts

Posted - 2002-08-07 : 01:50:41
Jai,

After merging back the words, do i have to follow the graz's method to use the like operator (like splitting in to words and search for each word in the database using like operator ) or is there any shortcut other than the above method ?

many thanks
thiru


Go to Top of Page

Tim
Starting Member

392 Posts

Posted - 2002-08-07 : 01:58:49
you could always cast the text columns to varchar... but you'd be limited to 8000 character strings. Dunno if that helps or not.

----
Nancy Davolio: Best looking chick at Northwind 1992-2000
Go to Top of Page

jasper_smith
SQL Server MVP &amp; SQLTeam MVY

846 Posts

Posted - 2002-08-07 : 04:30:52
Here's an example of server side search phrase parsing. This is more for Full Text Search - where you actually get errors if you don't eliminate noise words from the search phrase if you are ANDing the words together but maybe it will help you in your situation.


CREATE function fn_noise(@str varchar(200),@type varchar(3)='AND')
returns varchar(200)
as
/*
This function is a helper function for FTS search
It splits the search string (using another function fn_Split)
into words and then eliminates noise words by joining to
the noise table. The noise table ci_title_search_noise is an
import of the FTS noise.eng file and will need to be maintained
to stay in sync with the actual noise file for FTS
This function prevents errors when searches contain noise words
It can be used for AND and OR searches


Initial Creation J Smith 22 Mar 2002
*/
begin

declare @txt varchar(200)
declare @and char(7) ; set @and = '" '+@type+' "'
declare @t table(word varchar(50))

--Split search text into words - space separator
insert @t(word)
select value from dbo.fn_Split(@str,' ')

--remove noise words by joining to noise table
delete from @t
from @t t join ci_title_search_noise s (nolock)
on t.word=s.word

--reassemble string
SELECT @txt = COALESCE(@txt +'*'+ @and,'') + word FROM @t
SELECT @txt = '"'+@txt+'*"'

return @txt
end
go




fn_Split is available here
[url]http://www.sqlmag.com/Articles/Index.cfm?ArticleID=21071[/url]

HTH
Jasper Smith
Go to Top of Page

thiruna
Starting Member

41 Posts

Posted - 2002-08-08 : 00:11:16
thanks smith,

after eliminating the unwanted words, is it necessary to split into each word and use the like operator with a do while loop...(as like graz search) or any other alternative...

If i run a search for each word after eliminating the unwanted ones in a loop, i guess the performance will be much lower...is it?

many thanks
thiru


Go to Top of Page
   

- Advertisement -