数据库内核月报－ 2017/04 - MySQL · 引擎介绍 · Sphinx源码剖析（二） - 《数据库内核月报》

SPH文件生成

在本节中，我将会介绍索引文件sph的生成，从上一节我们得知sph文件保存了Sphinx的索引元信息以及一些索引相关的配置信息

SPH文件生成

先来看代码，其中sph文件的生成是在CSphIndex_VLN::WriteHeader这个函数中:


1. bool CSphIndex_VLN::WriteHeader ( const BuildHeader_t & tBuildHeader, CSphWriter & fdInfo ) const
2. {
3. // version
4. fdInfo.PutDword ( INDEX_MAGIC_HEADER );
5. fdInfo.PutDword ( INDEX_FORMAT_VERSION );

7. // bits
8. fdInfo.PutDword ( USE_64BIT );

10. // docinfo
11. fdInfo.PutDword ( m_tSettings.m_eDocinfo );

13. // schema
14. WriteSchema ( fdInfo, m_tSchema );

16. // min doc
17. fdInfo.PutOffset ( tBuildHeader.m_uMinDocid ); // was dword in v.1
18. if ( m_tSettings.m_eDocinfo==SPH_DOCINFO_INLINE )
19. fdInfo.PutBytes ( tBuildHeader.m_pMinRow, m_tSchema.GetRowSize()*sizeof(CSphRowitem) );

21. // wordlist checkpoints
22. fdInfo.PutOffset ( tBuildHeader.m_iDictCheckpointsOffset );
23. fdInfo.PutDword ( tBuildHeader.m_iDictCheckpoints );
24. fdInfo.PutByte ( tBuildHeader.m_iInfixCodepointBytes );
25. fdInfo.PutDword ( (DWORD)tBuildHeader.m_iInfixBlocksOffset );
26. fdInfo.PutDword ( tBuildHeader.m_iInfixBlocksWordsSize );

28. // index stats
29. fdInfo.PutDword ( (DWORD)tBuildHeader.m_iTotalDocuments ); // FIXME? we don't expect over 4G docs per just 1 local index
30. fdInfo.PutOffset ( tBuildHeader.m_iTotalBytes );
31. fdInfo.PutDword ( tBuildHeader.m_iTotalDups );

33. // index settings
34. SaveIndexSettings ( fdInfo, m_tSettings );

36. // tokenizer info
37. assert ( m_pTokenizer );
38. SaveTokenizerSettings ( fdInfo, m_pTokenizer, m_tSettings.m_iEmbeddedLimit );

40. // dictionary info
41. assert ( m_pDict );
42. SaveDictionarySettings ( fdInfo, m_pDict, false, m_tSettings.m_iEmbeddedLimit );

44. fdInfo.PutDword ( tBuildHeader.m_uKillListSize );
45. fdInfo.PutOffset ( tBuildHeader.m_iMinMaxIndex );

47. // field filter info
48. SaveFieldFilterSettings ( fdInfo, m_pFieldFilter );

50. // average field lengths
51. if ( m_tSettings.m_bIndexFieldLens )
52. ARRAY_FOREACH ( i, m_tSchema.m_dFields )
53. fdInfo.PutOffset ( m_dFieldLens[i] );

55. return true;
56. }

然后按顺序来解释下每一项字段的含义.

前两个字段INDEX_MAGIC_HEADER和INDEX_FORMAT_VERSION分别是magic number和索引版本号
第三个字段USE_64BIT表示是否使用64位的document和word id(默认是使用).
然后是写入docinfo,这个字段也就是配置中的docinfo字段(index block中)
接下来将会写入schema，也就是索引的schema信息，比如当前索引的字段名，当前需要建立的属性名等等.


1. void WriteSchema ( CSphWriter & fdInfo, const CSphSchema & tSchema )
2. {
3. // schema
4. fdInfo.PutDword ( tSchema.m_dFields.GetLength() );
5. ARRAY_FOREACH ( i, tSchema.m_dFields )
6. WriteSchemaColumn ( fdInfo, tSchema.m_dFields[i] );

8. fdInfo.PutDword ( tSchema.GetAttrsCount() );
9. for ( int i=0; i<tSchema.GetAttrsCount(); i++ )
10. WriteSchemaColumn ( fdInfo, tSchema.GetAttr(i) );
11. }

然后是写入当前索引集的最小doc id(m_uMinDocid)
接下来是根据docinfo(也就是属性存储)的配置来选择是否写入行信息(当docinfo为inline的话，表示attribute value 将会存储在spd文件中).
然后是写入wordlist的checkpoint.
然后是索引的统计信息(m_iTotalDocuments/m_iTotalBytes/m_iTotalDups).
接下来是写入对应的索引配置信息


1. void SaveIndexSettings ( CSphWriter & tWriter, const CSphIndexSettings & tSettings )
2. {
3. tWriter.PutDword ( tSettings.m_iMinPrefixLen );
4. tWriter.PutDword ( tSettings.m_iMinInfixLen );
5. tWriter.PutDword ( tSettings.m_iMaxSubstringLen );
6. tWriter.PutByte ( tSettings.m_bHtmlStrip ? 1 : 0 );
7. tWriter.PutString ( tSettings.m_sHtmlIndexAttrs.cstr () );
8. tWriter.PutString ( tSettings.m_sHtmlRemoveElements.cstr () );
9. tWriter.PutByte ( tSettings.m_bIndexExactWords ? 1 : 0 );
10. tWriter.PutDword ( tSettings.m_eHitless );
11. tWriter.PutDword ( tSettings.m_eHitFormat );
12. tWriter.PutByte ( tSettings.m_bIndexSP );
13. tWriter.PutString ( tSettings.m_sZones );
14. tWriter.PutDword ( tSettings.m_iBoundaryStep );
15. tWriter.PutDword ( tSettings.m_iStopwordStep );
16. tWriter.PutDword ( tSettings.m_iOvershortStep );
17. tWriter.PutDword ( tSettings.m_iEmbeddedLimit );
18. tWriter.PutByte ( tSettings.m_eBigramIndex );
19. tWriter.PutString ( tSettings.m_sBigramWords );
20. tWriter.PutByte ( tSettings.m_bIndexFieldLens );
21. tWriter.PutByte ( tSettings.m_eChineseRLP );
22. tWriter.PutString ( tSettings.m_sRLPContext );
23. tWriter.PutString ( tSettings.m_sIndexTokenFilter );
24. }

写入对应的tokenizer的配置信息,


1. void SaveTokenizerSettings ( CSphWriter & tWriter, ISphTokenizer * pTokenizer, int iEmbeddedLimit )
2. {
3. assert ( pTokenizer );

5. const CSphTokenizerSettings & tSettings = pTokenizer->GetSettings ();
6. tWriter.PutByte ( tSettings.m_iType );
7. tWriter.PutString ( tSettings.m_sCaseFolding.cstr () );
8. tWriter.PutDword ( tSettings.m_iMinWordLen );

10. bool bEmbedSynonyms = pTokenizer->GetSynFileInfo ().m_uSize<=(SphOffset_t)iEmbeddedLimit;
11. tWriter.PutByte ( bEmbedSynonyms ? 1 : 0 );
12. if ( bEmbedSynonyms )
13. pTokenizer->WriteSynonyms ( tWriter );

15. tWriter.PutString ( tSettings.m_sSynonymsFile.cstr () );
16. WriteFileInfo ( tWriter, pTokenizer->GetSynFileInfo () );
17. tWriter.PutString ( tSettings.m_sBoundary.cstr () );
18. tWriter.PutString ( tSettings.m_sIgnoreChars.cstr () );
19. tWriter.PutDword ( tSettings.m_iNgramLen );
20. tWriter.PutString ( tSettings.m_sNgramChars.cstr () );
21. tWriter.PutString ( tSettings.m_sBlendChars.cstr () );
22. tWriter.PutString ( tSettings.m_sBlendMode.cstr () );
23. }

写入dictionary的配置信息(比如stop word之类).


1. void SaveDictionarySettings ( CSphWriter & tWriter, CSphDict * pDict, bool bForceWordDict, int iEmbeddedLimit )
2. {
3. assert ( pDict );
4. const CSphDictSettings & tSettings = pDict->GetSettings ();

6. tWriter.PutString ( tSettings.m_sMorphology.cstr () );
7. .............................

9. bool bEmbedStopwords = uTotalSize<=(SphOffset_t)iEmbeddedLimit;
10. tWriter.PutByte ( bEmbedStopwords ? 1 : 0 );
11. if ( bEmbedStopwords )
12. pDict->WriteStopwords ( tWriter );

14. tWriter.PutString ( tSettings.m_sStopwords.cstr () );
15. tWriter.PutDword ( dSWFileInfos.GetLength () );
16. ARRAY_FOREACH ( i, dSWFileInfos )
17. {
18. tWriter.PutString ( dSWFileInfos[i].m_sFilename.cstr () );
19. WriteFileInfo ( tWriter, dSWFileInfos[i] );
20. }

22. const CSphVector <CSphSavedFile> & dWFFileInfos = pDict->GetWordformsFileInfos ();
23. uTotalSize = 0;
24. ARRAY_FOREACH ( i, dWFFileInfos )
25. uTotalSize += dWFFileInfos[i].m_uSize;

27. bool bEmbedWordforms = uTotalSize<=(SphOffset_t)iEmbeddedLimit;
28. tWriter.PutByte ( bEmbedWordforms ? 1 : 0 );
29. if ( bEmbedWordforms )
30. pDict->WriteWordforms ( tWriter );

32. tWriter.PutDword ( dWFFileInfos.GetLength() );
33. ARRAY_FOREACH ( i, dWFFileInfos )
34. {
35. tWriter.PutString ( dWFFileInfos[i].m_sFilename.cstr() );
36. WriteFileInfo ( tWriter, dWFFileInfos[i] );
37. }

39. tWriter.PutDword ( tSettings.m_iMinStemmingLen );
40. tWriter.PutByte ( tSettings.m_bWordDict || bForceWordDict );
41. tWriter.PutByte ( tSettings.m_bStopwordsUnstemmed );
42. tWriter.PutString ( pDict->GetMorphDataFingerprint() );
43. }

然后是写入killlist的size（m_uKillListSize)
写入m_iMinMaxIndex，这个选项也就是表示document size.


1. CSphFixedVector<CSphRowitem> dMinRow ( tNewSchema.GetRowSize() );
2. ...............
3. int iNewStride = DOCINFO_IDSIZE + tNewSchema.GetRowSize();

5. int64_t iNewMinMaxIndex = m_iDocinfo * iNewStride;
6. ..............................
7. tBuildHeader.m_iMinMaxIndex = iNewMinMaxIndex;

写入regex相关配置(regexp_filter)


1. void SaveFieldFilterSettings ( CSphWriter & tWriter, ISphFieldFilter * pFieldFilter )
2. {
3. if ( !pFieldFilter )
4. {
5. tWriter.PutDword ( 0 );
6. return;
7. }

9. CSphFieldFilterSettings tSettings;
10. pFieldFilter->GetSettings ( tSettings );

12. tWriter.PutDword ( tSettings.m_dRegexps.GetLength() );
13. ARRAY_FOREACH ( i, tSettings.m_dRegexps )
14. tWriter.PutString ( tSettings.m_dRegexps[i] );

16. tWriter.PutByte(1); // deprecated utf8 flag
17. }

最后是写入对应的schema field长度.

原文：http://mysql.taobao.org/monthly/2017/04/03/