Lucene 1.4.3 : IndexWriter.addDocument(doc) fails when run on OS requiring permissions

Hi!

I'm writing a java program that uses Lucene 1.4.3 to index
and create a vector file of words found in Text Files. The
purpose is for text mining.

I created a Java .Jar file from my program and my python
script calls the Java Jar executable. This is all triggered
by my DTML code.

I'm running on Linux and i have no problem executing the
script when i execute via command line. But once i trigger
the script via the web (using Zope/Plone external methods )
it doesn't work anymore. This is because of the strict
permissions that LInux has over its files and folders. 

I've narrowed down the problem to the
IndexWriter.addDocument(doc) method in Lucene 1.4.3 and as
you can see below my code fails specifically when a new
FieldsWriter object is being initialised.

I strongly suspect that it fails at this point but have no
idea how to overcome this problem. I know that it has to do
with the permissions because th eprogram works like a
miracle when it is called via command line by the super user
(sudo).

Could anyone give me any pointers or ideas of how i could
overcome this.

The final statement which is printed before the program
hangs is:
"Entering DocumentWriter.AddDocument.... (4)"

Here is the portions of my relevant code :



//----------------------------------------------------------
---------------------------------
//			Indexer.Java // This is my own method and class
//----------------------------------------------------------
---------------------------------
        // continued from some other code......

	Document doc = new Document();

        doc.add(Field.Text("articleTitle",
articleTitle, true));
        doc.add(Field.Text("articleURL",
articleURL, true));
        doc.add(Field.Text("articleSummary",
articleSummary, true));
        doc.add(Field.Text("articleDate",
articleDate, true));
        doc.add(Field.Text("articleSource",
articleSource, true));
        doc.add(Field.Text("articleBody",
articleBody, true));
        doc.add(Field.Keyword("filename",
f.getCanonicalPath()));
        
	try
                {
                    writer.addDocument(doc); // indexing
fails because this statement cannot be executed

                }

                catch (Exception e)

                {
                    System.err.println ("Cannot add doc
exception thrown!");

                }


//----------------------------------------------------------
---------------------------------
//			IndexWriter.Java // Lucene 1.4.3
//----------------------------------------------------------
---------------------------------


public void addDocument(Document doc) throws IOException {

      addDocument(doc, analyzer);
  }


public void addDocument(Document doc, Analyzer analyzer)
throws IOException {
    
    DocumentWriter dw;  
      
    dw = new DocumentWriter(ramDirectory, analyzer,
similarity, maxFieldLength);
    
    String segmentName = newSegmentName();
    dw.addDocument(segmentName, doc);      		// The program
fails to execute this line onwards!

    synchronized (this) {

      segmentInfos.addElement(new SegmentInfo(segmentName,
1, ramDirectory));
      maybeMergeSegments();
    }

  }


//----------------------------------------------------------
---------------------------------
//			DocumentWriter.Java // Lucene 1.4.3
//----------------------------------------------------------
---------------------------------



final void addDocument(String segment, Document doc)
            throws IOException {
   
      System.out.println("Entering
DocumentWriter.AddDocument.... (1)");
   
    // write field names
    fieldInfos = new FieldInfos();
      System.out.println("Entering
DocumentWriter.AddDocument.... (2)");

    fieldInfos.add(doc);
          System.out.println("Entering
DocumentWriter.AddDocument.... (3)");

    fieldInfos.write(directory, segment +
".fnm");

          System.out.println("Entering
DocumentWriter.AddDocument.... (4)");  // The program
fails after this 

    // write field values
    FieldsWriter fieldsWriter =
            new FieldsWriter(directory, segment,
fieldInfos);			// Program fails to execute this statement
    
          System.out.println("Entering
DocumentWriter.AddDocument.... (5)");

    try {
      fieldsWriter.addDocument(doc);

      System.out.println("Entering
DocumentWriter.AddDocument.... (6)");
      
    } finally {
      fieldsWriter.close();
            System.out.println("Entering
DocumentWriter.AddDocument.... (7)");

    }

          System.out.println("Entering
DocumentWriter.AddDocument.... (8)");

    // invert doc into postingTable
    postingTable.clear();			  // clear postingTable
    fieldLengths = new int[fieldInfos.size()];    // init
fieldLengths
    fieldPositions = new int[fieldInfos.size()];  // init
fieldPositions

          System.out.println("Entering
DocumentWriter.AddDocument.... (9)");

    fieldBoosts = new float[fieldInfos.size()];	  // init
fieldBoosts
    Arrays.fill(fieldBoosts, doc.getBoost());

          System.out.println("Entering
DocumentWriter.AddDocument.... (10)");

    invertDocument(doc);

          System.out.println("Entering
DocumentWriter.AddDocument.... (11)");

    // sort postingTable into an array
    Posting[] postings = sortPostingTable();

          System.out.println("Entering
DocumentWriter.AddDocument.... (12)");

    /*
    for (int i = 0; i < postings.length; i++) {
      Posting posting = postings[i];
      System.out.print(posting.term);
      System.out.print(" freq=" + posting.freq);
      System.out.print(" pos=");
      System.out.print(posting.positions[0]);
      for (int j = 1; j < posting.freq; j++)
	System.out.print("," + posting.positions[j]);
      System.out.println("");
    }
    */

    // write postings
    writePostings(postings, segment);

          System.out.println("Entering
DocumentWriter.AddDocument.... (13)");

    // write norms of indexed fields
    writeNorms(doc, segment);

          System.out.println("Entering
DocumentWriter.AddDocument.... (14)");

  }

//----------------------------------------------------------
---------------------------------
//			FieldsWriter.Java // Lucene 1.4.3
//----------------------------------------------------------
---------------------------------


  FieldsWriter(Directory d, String segment, FieldInfos fn)
       throws IOException {
    fieldInfos = fn;
    fieldsStream = d.createFile(segment +
".fdt");
    indexStream = d.createFile(segment + ".fdx");
  }

//----------------------------------------------------------
---------------------------------
//			FSDirectory.Java // Lucene 1.4.3
//----------------------------------------------------------
---------------------------------


public final OutputStream createFile(String name) throws
IOException {
  
      System.out.println("Entering
FSDirectory.createFile.... returning an
OutputStream");

    return new FSOutputStream(new File(directory, name));
  }

------------------------------------------------------------
---------
To unsubscribe, e-mail: java-user-unsubscribelucene.apache.org
For additional commands, e-mail: java-user-helplucene.apache.org

This is a very common use case and Lucene is most likely
not
the problem cause.

My guess is that (1) the first attempt to write anything to
disk failed. (2) opening the IndexWriter succeeded because
(a) the index exists already (from previous successful run)
and
(b) locks are maintained in /tmp or so.

You can try running the same way some simple code that just
writes something to a file in the same directory where the
index is maintained (also, check access permissions at that
folder).

Hope this helps,
Doron

(BTW, an exception stack trace would be far more informative
than
the print statements here - best catch the exception in the
top level and print it.)

Ridzwan Aminuddin <wansterinbox.com> wrote on
22/02/2007 00:20:12:

> Hi!
>
> I'm writing a java program that uses Lucene 1.4.3 to
index and
> create a vector file of words found in Text Files. The
purpose is
> for text mining.
>
> I created a Java .Jar file from my program and my
python script
> calls the Java Jar executable. This is all triggered by
my DTML code.
>
> I'm running on Linux and i have no problem executing
the script when
> i execute via command line. But once i trigger the
script via the
> web (using Zope/Plone external methods ) it doesn't
work anymore.
> This is because of the strict permissions that LInux
has over its
> files and folders.
>
> I've narrowed down the problem to the
IndexWriter.addDocument(doc)
> method in Lucene 1.4.3 and as you can see below my code
fails
> specifically when a new FieldsWriter object is being
initialised.
>
> I strongly suspect that it fails at this point but have
no idea how
> to overcome this problem. I know that it has to do with
the
> permissions because th eprogram works like a miracle
when it is
> called via command line by the super user (sudo).
>
> Could anyone give me any pointers or ideas of how i
could overcome this.
>
> The final statement which is printed before the program
hangs is:
> "Entering DocumentWriter.AddDocument....
(4)"
>
> Here is the portions of my relevant code :
>
>
>
>
//----------------------------------------------------------
---------------------------------

> //         Indexer.Java // This is my own method and
class
>
//----------------------------------------------------------
---------------------------------

>         // continued from some other code......
>
>    Document doc = new Document();
>
>         doc.add(Field.Text("articleTitle",
articleTitle, true));
>         doc.add(Field.Text("articleURL",
articleURL, true));
>         doc.add(Field.Text("articleSummary",
articleSummary, true));
>         doc.add(Field.Text("articleDate",
articleDate, true));
>         doc.add(Field.Text("articleSource",
articleSource, true));
>         doc.add(Field.Text("articleBody",
articleBody, true));
>         doc.add(Field.Keyword("filename",
f.getCanonicalPath()));
>
>    try
>                 {
>                     writer.addDocument(doc); //
indexing fails
> because this statement cannot be executed
>
>                 }
>
>                 catch (Exception e)
>
>                 {
>                     System.err.println ("Cannot
add doc exception
thrown!");
>
>                 }
>
>
>
//----------------------------------------------------------
---------------------------------

> //         IndexWriter.Java // Lucene 1.4.3
>
//----------------------------------------------------------
---------------------------------

>
>
> public void addDocument(Document doc) throws
IOException {
>
>       addDocument(doc, analyzer);
>   }
>
>
> public void addDocument(Document doc, Analyzer
analyzer) throws
IOException {
>
>     DocumentWriter dw;
>
>     dw = new DocumentWriter(ramDirectory, analyzer,
similarity,
> maxFieldLength);
>
>     String segmentName = newSegmentName();
>     dw.addDocument(segmentName, doc);            // The
program
> fails to execute this line onwards!
>
>     synchronized (this) {
>
>       segmentInfos.addElement(new
SegmentInfo(segmentName, 1,
ramDirectory));
>       maybeMergeSegments();
>     }
>
>   }
>
>
>
//----------------------------------------------------------
---------------------------------

> //         DocumentWriter.Java // Lucene 1.4.3
>
//----------------------------------------------------------
---------------------------------

>
>
>
> final void addDocument(String segment, Document doc)
>             throws IOException {
>
>       System.out.println("Entering
DocumentWriter.AddDocument.... (1)");
>
>     // write field names
>     fieldInfos = new FieldInfos();
>       System.out.println("Entering
DocumentWriter.AddDocument.... (2)");
>
>     fieldInfos.add(doc);
>           System.out.println("Entering
DocumentWriter.AddDocument....
(3)");
>
>     fieldInfos.write(directory, segment +
".fnm");
>
>           System.out.println("Entering
DocumentWriter.
> AddDocument.... (4)");  // The program fails after
this
>
>     // write field values
>     FieldsWriter fieldsWriter =
>             new FieldsWriter(directory, segment,
fieldInfos);
> // Program fails to execute this statement
>
>           System.out.println("Entering
DocumentWriter.AddDocument....
(5)");
>
>     try {
>       fieldsWriter.addDocument(doc);
>
>       System.out.println("Entering
DocumentWriter.AddDocument.... (6)");
>
>     } finally {
>       fieldsWriter.close();
>             System.out.println("Entering
DocumentWriter.AddDocument....
(7)");
>
>     }
>
>           System.out.println("Entering
DocumentWriter.AddDocument....
(8)");
>
>     // invert doc into postingTable
>     postingTable.clear();           // clear
postingTable
>     fieldLengths = new int[fieldInfos.size()];    //
init fieldLengths
>     fieldPositions = new int[fieldInfos.size()];  //
init fieldPositions
>
>           System.out.println("Entering
DocumentWriter.AddDocument....
(9)");
>
>     fieldBoosts = new float[fieldInfos.size()];     //
init fieldBoosts
>     Arrays.fill(fieldBoosts, doc.getBoost());
>
>           System.out.println("Entering
DocumentWriter.AddDocument....
(10)");
>
>     invertDocument(doc);
>
>           System.out.println("Entering
DocumentWriter.AddDocument....
(11)");
>
>     // sort postingTable into an array
>     Posting[] postings = sortPostingTable();
>
>           System.out.println("Entering
DocumentWriter.AddDocument....
(12)");
>
>     /*
>     for (int i = 0; i < postings.length; i++) {
>       Posting posting = postings[i];
>       System.out.print(posting.term);
>       System.out.print(" freq=" +
posting.freq);
>       System.out.print(" pos=");
>       System.out.print(posting.positions[0]);
>       for (int j = 1; j < posting.freq; j++)
>    System.out.print("," +
posting.positions[j]);
>       System.out.println("");
>     }
>     */
>
>     // write postings
>     writePostings(postings, segment);
>
>           System.out.println("Entering
DocumentWriter.AddDocument....
(13)");
>
>     // write norms of indexed fields
>     writeNorms(doc, segment);
>
>           System.out.println("Entering
DocumentWriter.AddDocument....
(14)");
>
>   }
>
>
//----------------------------------------------------------
---------------------------------

> //         FieldsWriter.Java // Lucene 1.4.3
>
//----------------------------------------------------------
---------------------------------

>
>
>   FieldsWriter(Directory d, String segment, FieldInfos
fn)
>        throws IOException {
>     fieldInfos = fn;
>     fieldsStream = d.createFile(segment +
".fdt");
>     indexStream = d.createFile(segment +
".fdx");
>   }
>
>
//----------------------------------------------------------
---------------------------------

> //         FSDirectory.Java // Lucene 1.4.3
>
//----------------------------------------------------------
---------------------------------

>
>
> public final OutputStream createFile(String name)
throws IOException {
>
>       System.out.println("Entering
FSDirectory.createFile....
> returning an OutputStream");
>
>     return new FSOutputStream(new File(directory,
name));
>   }
>
>
------------------------------------------------------------
---------
> To unsubscribe, e-mail: java-user-unsubscribelucene.apache.org
> For additional commands, e-mail: java-user-helplucene.apache.org
>


------------------------------------------------------------
---------
To unsubscribe, e-mail: java-user-unsubscribelucene.apache.org
For additional commands, e-mail: java-user-helplucene.apache.org

Is your disk almost full?  Under Linux, when you reach about
90% used on 
a file system, only the superuser can allocate more space
(e.g. create 
files, add data to files, etc.).

--MDC

------------------------------------------------------------
---------
To unsubscribe, e-mail: java-user-unsubscribelucene.apache.org
For additional commands, e-mail: java-user-helplucene.apache.org

This sounds like it has absolutely nothing to do with
Lucene, and
everything to do with good security permissions -- your
Zope/python front
end is most likely running as a user thta does not have
write permissions
to the directory where your index lives.  you'll need to
remedy that.

you can write a simple java app that doens't use lucene at
all -- just
creates a file and writes  "hellow world" to it --
and you will most
likely see this exact same behavior, dealing with teh file
permissions is
totally out side the scope of Lucene.


: Date: Thu, 22 Feb 2007 00:20:12 -0800
: From: Ridzwan Aminuddin <wansterinbox.com>
: Reply-To: java-userlucene.apache.org
: To: java-userlucene.apache.org
: Subject: Lucene 1.4.3 : IndexWriter.addDocument(doc) fails
when run on OS
:      requiring permissions
:
: Hi!
:
: I'm writing a java program that uses Lucene 1.4.3 to index
and create a vector file of words found in Text Files. The
purpose is for text mining.
:
: I created a Java .Jar file from my program and my python
script calls the Java Jar executable. This is all triggered
by my DTML code.
:
: I'm running on Linux and i have no problem executing the
script when i execute via command line. But once i trigger
the script via the web (using Zope/Plone external methods )
it doesn't work anymore. This is because of the strict
permissions that LInux has over its files and folders.
:
: I've narrowed down the problem to the
IndexWriter.addDocument(doc) method in Lucene 1.4.3 and as
you can see below my code fails specifically when a new
FieldsWriter object is being initialised.
:
: I strongly suspect that it fails at this point but have no
idea how to overcome this problem. I know that it has to do
with the permissions because th eprogram works like a
miracle when it is called via command line by the super user
(sudo).
:
: Could anyone give me any pointers or ideas of how i could
overcome this.
:
: The final statement which is printed before the program
hangs is:
: "Entering DocumentWriter.AddDocument.... (4)"
:
: Here is the portions of my relevant code :
:
:
:
:
//----------------------------------------------------------
---------------------------------
: //			Indexer.Java // This is my own method and class
:
//----------------------------------------------------------
---------------------------------
:         // continued from some other code......
:
: 	Document doc = new Document();
:
:         doc.add(Field.Text("articleTitle",
articleTitle, true));
:         doc.add(Field.Text("articleURL",
articleURL, true));
:         doc.add(Field.Text("articleSummary",
articleSummary, true));
:         doc.add(Field.Text("articleDate",
articleDate, true));
:         doc.add(Field.Text("articleSource",
articleSource, true));
:         doc.add(Field.Text("articleBody",
articleBody, true));
:         doc.add(Field.Keyword("filename",
f.getCanonicalPath()));
:
: 	try
:                 {
:                     writer.addDocument(doc); // indexing
fails because this statement cannot be executed
:
:                 }
:
:                 catch (Exception e)
:
:                 {
:                     System.err.println ("Cannot add
doc exception thrown!");
:
:                 }
:
:
:
//----------------------------------------------------------
---------------------------------
: //			IndexWriter.Java // Lucene 1.4.3
:
//----------------------------------------------------------
---------------------------------
:
:
: public void addDocument(Document doc) throws IOException
{
:
:       addDocument(doc, analyzer);
:   }
:
:
: public void addDocument(Document doc, Analyzer analyzer)
throws IOException {
:
:     DocumentWriter dw;
:
:     dw = new DocumentWriter(ramDirectory, analyzer,
similarity, maxFieldLength);
:
:     String segmentName = newSegmentName();
:     dw.addDocument(segmentName, doc);      		// The
program fails to execute this line onwards!
:
:     synchronized (this) {
:
:       segmentInfos.addElement(new SegmentInfo(segmentName,
1, ramDirectory));
:       maybeMergeSegments();
:     }
:
:   }
:
:
:
//----------------------------------------------------------
---------------------------------
: //			DocumentWriter.Java // Lucene 1.4.3
:
//----------------------------------------------------------
---------------------------------
:
:
:
: final void addDocument(String segment, Document doc)
:             throws IOException {
:
:       System.out.println("Entering
DocumentWriter.AddDocument.... (1)");
:
:     // write field names
:     fieldInfos = new FieldInfos();
:       System.out.println("Entering
DocumentWriter.AddDocument.... (2)");
:
:     fieldInfos.add(doc);
:           System.out.println("Entering
DocumentWriter.AddDocument.... (3)");
:
:     fieldInfos.write(directory, segment +
".fnm");
:
:           System.out.println("Entering
DocumentWriter.AddDocument.... (4)");  // The program
fails after this
:
:     // write field values
:     FieldsWriter fieldsWriter =
:             new FieldsWriter(directory, segment,
fieldInfos);			// Program fails to execute this statement
:
:           System.out.println("Entering
DocumentWriter.AddDocument.... (5)");
:
:     try {
:       fieldsWriter.addDocument(doc);
:
:       System.out.println("Entering
DocumentWriter.AddDocument.... (6)");
:
:     } finally {
:       fieldsWriter.close();
:             System.out.println("Entering
DocumentWriter.AddDocument.... (7)");
:
:     }
:
:           System.out.println("Entering
DocumentWriter.AddDocument.... (8)");
:
:     // invert doc into postingTable
:     postingTable.clear();			  // clear postingTable
:     fieldLengths = new int[fieldInfos.size()];    // init
fieldLengths
:     fieldPositions = new int[fieldInfos.size()];  // init
fieldPositions
:
:           System.out.println("Entering
DocumentWriter.AddDocument.... (9)");
:
:     fieldBoosts = new float[fieldInfos.size()];	  // init
fieldBoosts
:     Arrays.fill(fieldBoosts, doc.getBoost());
:
:           System.out.println("Entering
DocumentWriter.AddDocument.... (10)");
:
:     invertDocument(doc);
:
:           System.out.println("Entering
DocumentWriter.AddDocument.... (11)");
:
:     // sort postingTable into an array
:     Posting[] postings = sortPostingTable();
:
:           System.out.println("Entering
DocumentWriter.AddDocument.... (12)");
:
:     /*
:     for (int i = 0; i < postings.length; i++) {
:       Posting posting = postings[i];
:       System.out.print(posting.term);
:       System.out.print(" freq=" +
posting.freq);
:       System.out.print(" pos=");
:       System.out.print(posting.positions[0]);
:       for (int j = 1; j < posting.freq; j++)
: 	System.out.print("," + posting.positions[j]);
:       System.out.println("");
:     }
:     */
:
:     // write postings
:     writePostings(postings, segment);
:
:           System.out.println("Entering
DocumentWriter.AddDocument.... (13)");
:
:     // write norms of indexed fields
:     writeNorms(doc, segment);
:
:           System.out.println("Entering
DocumentWriter.AddDocument.... (14)");
:
:   }
:
:
//----------------------------------------------------------
---------------------------------
: //			FieldsWriter.Java // Lucene 1.4.3
:
//----------------------------------------------------------
---------------------------------
:
:
:   FieldsWriter(Directory d, String segment, FieldInfos
fn)
:        throws IOException {
:     fieldInfos = fn;
:     fieldsStream = d.createFile(segment +
".fdt");
:     indexStream = d.createFile(segment +
".fdx");
:   }
:
:
//----------------------------------------------------------
---------------------------------
: //			FSDirectory.Java // Lucene 1.4.3
:
//----------------------------------------------------------
---------------------------------
:
:
: public final OutputStream createFile(String name) throws
IOException {
:
:       System.out.println("Entering
FSDirectory.createFile.... returning an
OutputStream");
:
:     return new FSOutputStream(new File(directory, name));
:   }
:
:
------------------------------------------------------------
---------
: To unsubscribe, e-mail: java-user-unsubscribelucene.apache.org
: For additional commands, e-mail: java-user-helplucene.apache.org
:



-Hoss


------------------------------------------------------------
---------
To unsubscribe, e-mail: java-user-unsubscribelucene.apache.org
For additional commands, e-mail: java-user-helplucene.apache.org

Hi Guys.

Ok thanks for the replies. You guys are right that it is to
do with the system and not with Lucene. However, what i'm
trying to do is to pinpoint and narrow down to the exact
place that causes the system to fail. and then from there
try to remedy the problem. 

The odd thing is that the program is still able to write
other files to the subdirectories that the program itself
creates. It is only when it goes through this indexing
process that this program halts due to the insufficient
permissions. But the directory i provided to store the
target index files has been set to read/write/execute
(drwxrwxrwx) all permissions. 

In any case, i suspect that it is due to this portion of
code.


:   FieldsWriter(Directory d, String segment, FieldInfos
fn)
:        throws IOException {
:     fieldInfos = fn;
:     fieldsStream = d.createFile(segment +
".fdt");
:     indexStream = d.createFile(segment +
".fdx");
:   }


Are these two files created in the Directory d?
And is this Directory d the ramDirectory that i have
provided when i called the method: dw = new
DocumentWriter(ramDirectory, analyzer, similarity,
 maxFieldLength);
in IndexWriter.Java?

If yes. then where exactly does this ramDirectory point to?
cos as far as i can see from the code, ramDirectory is
initialised as 

RamDirectory ramDirectory = new RamDirectory()

I think the program fails when it tries to create these two
files somehow.

Please help to enlighten me?... Maybe knowing the exact path
to this ramDirectory would help me in finding out which
folder i need to provide access to....

Also, does Lucene ever write any temp data to /tmp ?



> -----Original Message-----
> From: hossman_lucenefucit.org
> Sent: Thu, 22 Feb 2007 11:43:38 -0800 (PST)
> To: java-userlucene.apache.org
> Subject: Re: Lucene 1.4.3 :
IndexWriter.addDocument(doc) fails when run
> on OS requiring permissions
> 
> 
> This sounds like it has absolutely nothing to do with
Lucene, and
> everything to do with good security permissions -- your
Zope/python front
> end is most likely running as a user thta does not have
write permissions
> to the directory where your index lives.  you'll need
to remedy that.
> 
> you can write a simple java app that doens't use lucene
at all -- just
> creates a file and writes  "hellow world" to
it -- and you will most
> likely see this exact same behavior, dealing with teh
file permissions is
> totally out side the scope of Lucene.
> 
> 
> : Date: Thu, 22 Feb 2007 00:20:12 -0800
> : From: Ridzwan Aminuddin <wansterinbox.com>
> : Reply-To: java-userlucene.apache.org
> : To: java-userlucene.apache.org
> : Subject: Lucene 1.4.3 : IndexWriter.addDocument(doc)
fails when run on
> OS
> :      requiring permissions
> :
> : Hi!
> :
> : I'm writing a java program that uses Lucene 1.4.3 to
index and create a
> vector file of words found in Text Files. The purpose
is for text mining.
> :
> : I created a Java .Jar file from my program and my
python script calls
> the Java Jar executable. This is all triggered by my
DTML code.
> :
> : I'm running on Linux and i have no problem executing
the script when i
> execute via command line. But once i trigger the script
via the web
> (using Zope/Plone external methods ) it doesn't work
anymore. This is
> because of the strict permissions that LInux has over
its files and
> folders.
> :
> : I've narrowed down the problem to the
IndexWriter.addDocument(doc)
> method in Lucene 1.4.3 and as you can see below my code
fails
> specifically when a new FieldsWriter object is being
initialised.
> :
> : I strongly suspect that it fails at this point but
have no idea how to
> overcome this problem. I know that it has to do with
the permissions
> because th eprogram works like a miracle when it is
called via command
> line by the super user (sudo).
> :
> : Could anyone give me any pointers or ideas of how i
could overcome
> this.
> :
> : The final statement which is printed before the
program hangs is:
> : "Entering DocumentWriter.AddDocument....
(4)"
> :
> : Here is the portions of my relevant code :
> :
> :
> :
> :
>
//----------------------------------------------------------
---------------------------------
> : //			Indexer.Java // This is my own method and class
> :
>
//----------------------------------------------------------
---------------------------------
> :         // continued from some other code......
> :
> : 	Document doc = new Document();
> :
> :         doc.add(Field.Text("articleTitle",
articleTitle, true));
> :         doc.add(Field.Text("articleURL",
articleURL, true));
> :        
doc.add(Field.Text("articleSummary",
articleSummary, true));
> :         doc.add(Field.Text("articleDate",
articleDate, true));
> :         doc.add(Field.Text("articleSource",
articleSource, true));
> :         doc.add(Field.Text("articleBody",
articleBody, true));
> :         doc.add(Field.Keyword("filename",
f.getCanonicalPath()));
> :
> : 	try
> :                 {
> :                     writer.addDocument(doc); //
indexing fails because
> this statement cannot be executed
> :
> :                 }
> :
> :                 catch (Exception e)
> :
> :                 {
> :                     System.err.println ("Cannot
add doc exception
> thrown!");
> :
> :                 }
> :
> :
> :
>
//----------------------------------------------------------
---------------------------------
> : //			IndexWriter.Java // Lucene 1.4.3
> :
>
//----------------------------------------------------------
---------------------------------
> :
> :
> : public void addDocument(Document doc) throws
IOException {
> :
> :       addDocument(doc, analyzer);
> :   }
> :
> :
> : public void addDocument(Document doc, Analyzer
analyzer) throws
> IOException {
> :
> :     DocumentWriter dw;
> :
> :     dw = new DocumentWriter(ramDirectory, analyzer,
similarity,
> maxFieldLength);
> :
> :     String segmentName = newSegmentName();
> :     dw.addDocument(segmentName, doc);      		// The
program fails to
> execute this line onwards!
> :
> :     synchronized (this) {
> :
> :       segmentInfos.addElement(new
SegmentInfo(segmentName, 1,
> ramDirectory));
> :       maybeMergeSegments();
> :     }
> :
> :   }
> :
> :
> :
>
//----------------------------------------------------------
---------------------------------
> : //			DocumentWriter.Java // Lucene 1.4.3
> :
>
//----------------------------------------------------------
---------------------------------
> :
> :
> :
> : final void addDocument(String segment, Document doc)
> :             throws IOException {
> :
> :       System.out.println("Entering
DocumentWriter.AddDocument....
> (1)");
> :
> :     // write field names
> :     fieldInfos = new FieldInfos();
> :       System.out.println("Entering
DocumentWriter.AddDocument....
> (2)");
> :
> :     fieldInfos.add(doc);
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (3)");
> :
> :     fieldInfos.write(directory, segment +
".fnm");
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (4)");  // The program fails after this
> :
> :     // write field values
> :     FieldsWriter fieldsWriter =
> :             new FieldsWriter(directory, segment,
fieldInfos);			//
> Program fails to execute this statement
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (5)");
> :
> :     try {
> :       fieldsWriter.addDocument(doc);
> :
> :       System.out.println("Entering
DocumentWriter.AddDocument....
> (6)");
> :
> :     } finally {
> :       fieldsWriter.close();
> :             System.out.println("Entering
DocumentWriter.AddDocument....
> (7)");
> :
> :     }
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (8)");
> :
> :     // invert doc into postingTable
> :     postingTable.clear();			  // clear postingTable
> :     fieldLengths = new int[fieldInfos.size()];    //
init fieldLengths
> :     fieldPositions = new int[fieldInfos.size()];  //
init
> fieldPositions
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (9)");
> :
> :     fieldBoosts = new float[fieldInfos.size()];	  //
init fieldBoosts
> :     Arrays.fill(fieldBoosts, doc.getBoost());
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (10)");
> :
> :     invertDocument(doc);
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (11)");
> :
> :     // sort postingTable into an array
> :     Posting[] postings = sortPostingTable();
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (12)");
> :
> :     /*
> :     for (int i = 0; i < postings.length; i++) {
> :       Posting posting = postings[i];
> :       System.out.print(posting.term);
> :       System.out.print(" freq=" +
posting.freq);
> :       System.out.print(" pos=");
> :       System.out.print(posting.positions[0]);
> :       for (int j = 1; j < posting.freq; j++)
> : 	System.out.print("," +
posting.positions[j]);
> :       System.out.println("");
> :     }
> :     */
> :
> :     // write postings
> :     writePostings(postings, segment);
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (13)");
> :
> :     // write norms of indexed fields
> :     writeNorms(doc, segment);
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (14)");
> :
> :   }
> :
> :
>
//----------------------------------------------------------
---------------------------------
> : //			FieldsWriter.Java // Lucene 1.4.3
> :
>
//----------------------------------------------------------
---------------------------------
> :
> :
> :   FieldsWriter(Directory d, String segment,
FieldInfos fn)
> :        throws IOException {
> :     fieldInfos = fn;
> :     fieldsStream = d.createFile(segment +
".fdt");
> :     indexStream = d.createFile(segment +
".fdx");
> :   }
> :
> :
>
//----------------------------------------------------------
---------------------------------
> : //			FSDirectory.Java // Lucene 1.4.3
> :
>
//----------------------------------------------------------
---------------------------------
> :
> :
> : public final OutputStream createFile(String name)
throws IOException {
> :
> :       System.out.println("Entering
FSDirectory.createFile.... returning
> an OutputStream");
> :
> :     return new FSOutputStream(new File(directory,
name));
> :   }
> :
> :
------------------------------------------------------------
---------
> : To unsubscribe, e-mail: java-user-unsubscribelucene.apache.org
> : For additional commands, e-mail: java-user-helplucene.apache.org
> :
> 
> 
> 
> -Hoss
> 
> 
>
------------------------------------------------------------
---------
> To unsubscribe, e-mail: java-user-unsubscribelucene.apache.org
> For additional commands, e-mail: java-user-helplucene.apache.org

------------------------------------------------------------
---------
To unsubscribe, e-mail: java-user-unsubscribelucene.apache.org
For additional commands, e-mail: java-user-helplucene.apache.org

The easiest way to pin this down is to get the backtrace
from the
exception, e.g., e.printStackTrace(). That would tell a
lot.

That said, prior to 2.1, lucene would put lock files outside
the index
directory. I don't know if that's what you're hitting,
though, because I
think the writer should have taken the lock when you created
it, not
when you tried to do the addDocument.

-----Original Message-----
From: Ridzwan Aminuddin [mailto:wansterinbox.com] 
Sent: Thursday, February 22, 2007 8:48 PM
To: java-userlucene.apache.org
Subject: Re: Lucene 1.4.3 : IndexWriter.addDocument(doc)
fails when run
on OS requiring permissions

Hi Guys.

Ok thanks for the replies. You guys are right that it is to
do with the
system and not with Lucene. However, what i'm trying to do
is to
pinpoint and narrow down to the exact place that causes the
system to
fail. and then from there try to remedy the problem. 

The odd thing is that the program is still able to write
other files to
the subdirectories that the program itself creates. It is
only when it
goes through this indexing process that this program halts
due to the
insufficient permissions. But the directory i provided to
store the
target index files has been set to read/write/execute
(drwxrwxrwx) all
permissions. 

In any case, i suspect that it is due to this portion of
code.


:   FieldsWriter(Directory d, String segment, FieldInfos
fn)
:        throws IOException {
:     fieldInfos = fn;
:     fieldsStream = d.createFile(segment +
".fdt");
:     indexStream = d.createFile(segment +
".fdx");
:   }


Are these two files created in the Directory d?
And is this Directory d the ramDirectory that i have
provided when i
called the method: dw = new DocumentWriter(ramDirectory,
analyzer,
similarity,
 maxFieldLength);
in IndexWriter.Java?

If yes. then where exactly does this ramDirectory point to?
cos as far
as i can see from the code, ramDirectory is initialised as 

RamDirectory ramDirectory = new RamDirectory()

I think the program fails when it tries to create these two
files
somehow.

Please help to enlighten me?... Maybe knowing the exact path
to this
ramDirectory would help me in finding out which folder i
need to provide
access to....

Also, does Lucene ever write any temp data to /tmp ?



> -----Original Message-----
> From: hossman_lucenefucit.org
> Sent: Thu, 22 Feb 2007 11:43:38 -0800 (PST)
> To: java-userlucene.apache.org
> Subject: Re: Lucene 1.4.3 :
IndexWriter.addDocument(doc) fails when
run
> on OS requiring permissions
> 
> 
> This sounds like it has absolutely nothing to do with
Lucene, and
> everything to do with good security permissions -- your
Zope/python
front
> end is most likely running as a user thta does not have
write
permissions
> to the directory where your index lives.  you'll need
to remedy that.
> 
> you can write a simple java app that doens't use lucene
at all -- just
> creates a file and writes  "hellow world" to
it -- and you will most
> likely see this exact same behavior, dealing with teh
file permissions
is
> totally out side the scope of Lucene.
> 
> 
> : Date: Thu, 22 Feb 2007 00:20:12 -0800
> : From: Ridzwan Aminuddin <wansterinbox.com>
> : Reply-To: java-userlucene.apache.org
> : To: java-userlucene.apache.org
> : Subject: Lucene 1.4.3 : IndexWriter.addDocument(doc)
fails when run
on
> OS
> :      requiring permissions
> :
> : Hi!
> :
> : I'm writing a java program that uses Lucene 1.4.3 to
index and
create a
> vector file of words found in Text Files. The purpose
is for text
mining.
> :
> : I created a Java .Jar file from my program and my
python script
calls
> the Java Jar executable. This is all triggered by my
DTML code.
> :
> : I'm running on Linux and i have no problem executing
the script when
i
> execute via command line. But once i trigger the script
via the web
> (using Zope/Plone external methods ) it doesn't work
anymore. This is
> because of the strict permissions that LInux has over
its files and
> folders.
> :
> : I've narrowed down the problem to the
IndexWriter.addDocument(doc)
> method in Lucene 1.4.3 and as you can see below my code
fails
> specifically when a new FieldsWriter object is being
initialised.
> :
> : I strongly suspect that it fails at this point but
have no idea how
to
> overcome this problem. I know that it has to do with
the permissions
> because th eprogram works like a miracle when it is
called via command
> line by the super user (sudo).
> :
> : Could anyone give me any pointers or ideas of how i
could overcome
> this.
> :
> : The final statement which is printed before the
program hangs is:
> : "Entering DocumentWriter.AddDocument....
(4)"
> :
> : Here is the portions of my relevant code :
> :
> :
> :
> :
>
//----------------------------------------------------------
------------
---------------------
> : //			Indexer.Java // This is my own method and class
> :
>
//----------------------------------------------------------
------------
---------------------
> :         // continued from some other code......
> :
> : 	Document doc = new Document();
> :
> :         doc.add(Field.Text("articleTitle",
articleTitle, true));
> :         doc.add(Field.Text("articleURL",
articleURL, true));
> :        
doc.add(Field.Text("articleSummary",
articleSummary, true));
> :         doc.add(Field.Text("articleDate",
articleDate, true));
> :         doc.add(Field.Text("articleSource",
articleSource, true));
> :         doc.add(Field.Text("articleBody",
articleBody, true));
> :         doc.add(Field.Keyword("filename",
f.getCanonicalPath()));
> :
> : 	try
> :                 {
> :                     writer.addDocument(doc); //
indexing fails
because
> this statement cannot be executed
> :
> :                 }
> :
> :                 catch (Exception e)
> :
> :                 {
> :                     System.err.println ("Cannot
add doc exception
> thrown!");
> :
> :                 }
> :
> :
> :
>
//----------------------------------------------------------
------------
---------------------
> : //			IndexWriter.Java // Lucene 1.4.3
> :
>
//----------------------------------------------------------
------------
---------------------
> :
> :
> : public void addDocument(Document doc) throws
IOException {
> :
> :       addDocument(doc, analyzer);
> :   }
> :
> :
> : public void addDocument(Document doc, Analyzer
analyzer) throws
> IOException {
> :
> :     DocumentWriter dw;
> :
> :     dw = new DocumentWriter(ramDirectory, analyzer,
similarity,
> maxFieldLength);
> :
> :     String segmentName = newSegmentName();
> :     dw.addDocument(segmentName, doc);      		// The
program
fails to
> execute this line onwards!
> :
> :     synchronized (this) {
> :
> :       segmentInfos.addElement(new
SegmentInfo(segmentName, 1,
> ramDirectory));
> :       maybeMergeSegments();
> :     }
> :
> :   }
> :
> :
> :
>
//----------------------------------------------------------
------------
---------------------
> : //			DocumentWriter.Java // Lucene 1.4.3
> :
>
//----------------------------------------------------------
------------
---------------------
> :
> :
> :
> : final void addDocument(String segment, Document doc)
> :             throws IOException {
> :
> :       System.out.println("Entering
DocumentWriter.AddDocument....
> (1)");
> :
> :     // write field names
> :     fieldInfos = new FieldInfos();
> :       System.out.println("Entering
DocumentWriter.AddDocument....
> (2)");
> :
> :     fieldInfos.add(doc);
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (3)");
> :
> :     fieldInfos.write(directory, segment +
".fnm");
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (4)");  // The program fails after this
> :
> :     // write field values
> :     FieldsWriter fieldsWriter =
> :             new FieldsWriter(directory, segment,
fieldInfos);
//
> Program fails to execute this statement
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (5)");
> :
> :     try {
> :       fieldsWriter.addDocument(doc);
> :
> :       System.out.println("Entering
DocumentWriter.AddDocument....
> (6)");
> :
> :     } finally {
> :       fieldsWriter.close();
> :             System.out.println("Entering
DocumentWriter.AddDocument....
> (7)");
> :
> :     }
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (8)");
> :
> :     // invert doc into postingTable
> :     postingTable.clear();			  // clear postingTable
> :     fieldLengths = new int[fieldInfos.size()];    //
init
fieldLengths
> :     fieldPositions = new int[fieldInfos.size()];  //
init
> fieldPositions
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (9)");
> :
> :     fieldBoosts = new float[fieldInfos.size()];	  //
init
fieldBoosts
> :     Arrays.fill(fieldBoosts, doc.getBoost());
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (10)");
> :
> :     invertDocument(doc);
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (11)");
> :
> :     // sort postingTable into an array
> :     Posting[] postings = sortPostingTable();
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (12)");
> :
> :     /*
> :     for (int i = 0; i < postings.length; i++) {
> :       Posting posting = postings[i];
> :       System.out.print(posting.term);
> :       System.out.print(" freq=" +
posting.freq);
> :       System.out.print(" pos=");
> :       System.out.print(posting.positions[0]);
> :       for (int j = 1; j < posting.freq; j++)
> : 	System.out.print("," +
posting.positions[j]);
> :       System.out.println("");
> :     }
> :     */
> :
> :     // write postings
> :     writePostings(postings, segment);
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (13)");
> :
> :     // write norms of indexed fields
> :     writeNorms(doc, segment);
> :
> :           System.out.println("Entering
DocumentWriter.AddDocument....
> (14)");
> :
> :   }
> :
> :
>
//----------------------------------------------------------
------------
---------------------
> : //			FieldsWriter.Java // Lucene 1.4.3
> :
>
//----------------------------------------------------------
------------
---------------------
> :
> :
> :   FieldsWriter(Directory d, String segment,
FieldInfos fn)
> :        throws IOException {
> :     fieldInfos = fn;
> :     fieldsStream = d.createFile(segment +
".fdt");
> :     indexStream = d.createFile(segment +
".fdx");
> :   }
> :
> :
>
//----------------------------------------------------------
------------
---------------------
> : //			FSDirectory.Java // Lucene 1.4.3
> :
>
//----------------------------------------------------------
------------
---------------------
> :
> :
> : public final OutputStream createFile(String name)
throws IOException
{
> :
> :       System.out.println("Entering
FSDirectory.createFile....
returning
> an OutputStream");
> :
> :     return new FSOutputStream(new File(directory,
name));
> :   }
> :
> :
------------------------------------------------------------
---------
> : To unsubscribe, e-mail: java-user-unsubscribelucene.apache.org
> : For additional commands, e-mail: java-user-helplucene.apache.org
> :
> 
> 
> 
> -Hoss
> 
> 
>
------------------------------------------------------------
---------
> To unsubscribe, e-mail: java-user-unsubscribelucene.apache.org
> For additional commands, e-mail: java-user-helplucene.apache.org

------------------------------------------------------------
---------
To unsubscribe, e-mail: java-user-unsubscribelucene.apache.org
For additional commands, e-mail: java-user-helplucene.apache.org


------------------------------------------------------------
---------
To unsubscribe, e-mail: java-user-unsubscribelucene.apache.org
For additional commands, e-mail: java-user-helplucene.apache.org