|
| 1 | +package main |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "io" |
| 6 | + |
| 7 | + "github.com/dcadenas/pagerank" |
| 8 | + "github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/conversion" |
| 9 | + "github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/conversion/datastructures" |
| 10 | +) |
| 11 | + |
| 12 | +const ( |
| 13 | + // Chance of following a link rather than jumping randomly. |
| 14 | + FOLLOW_LINK_CHANCE = 0.85 |
| 15 | + // Smaller number here yields a more exact result; more CPU cycles required. |
| 16 | + TOLERANCE = 0.00001 |
| 17 | +) |
| 18 | + |
| 19 | +type Document struct { |
| 20 | + docId int |
| 21 | + filePath string |
| 22 | + rank float64 |
| 23 | +} |
| 24 | + |
| 25 | +// Read LSIF data from the given reader and return the files from the |
| 26 | +// index along with their pagerank. The results are not sorted. |
| 27 | +func PageRankLsif(indexFile io.Reader) (*[]Document, error) { |
| 28 | + // Build a conversion.State representing the input lsif index. |
| 29 | + state, err := conversion.CorrelateInMemory(context.TODO(), indexFile, "", nil) |
| 30 | + if err != nil { |
| 31 | + return nil, err |
| 32 | + } |
| 33 | + |
| 34 | + edges := addReferenceEdges(state) |
| 35 | + |
| 36 | + if *addImplEdges { |
| 37 | + addImplementationEdges(state, edges) |
| 38 | + } |
| 39 | + |
| 40 | + return runPageRanker(state, edges), nil |
| 41 | +} |
| 42 | + |
| 43 | +func addReferenceEdges(state *conversion.State) *map[int]int { |
| 44 | + // First, we need a map of range id -> doc id for the whole index. |
| 45 | + // Even for a very large index it's likely only to be a few million entries. |
| 46 | + // Since we're the only use case needing this lookup, we build it ephemerally. |
| 47 | + // TODO(stevey): Can we get this info from State without preprocessing? |
| 48 | + rangeToDoc := make(map[int]int) |
| 49 | + state.Contains.Each(func(docId int, rangeIDs *datastructures.IDSet) { |
| 50 | + rangeIDs.Each(func(rangeId int) { |
| 51 | + rangeToDoc[rangeId] = docId |
| 52 | + }) |
| 53 | + }) |
| 54 | + |
| 55 | + // Walk the references and find each one's file, and also the file(s) |
| 56 | + // containing the definition being referenced. These two files make an edge. |
| 57 | + // We omit links from files referencing themselves because PageRank ignores them. |
| 58 | + edges := make(map[int]int) |
| 59 | + for _, documentMap := range state.ReferenceData { |
| 60 | + documentMap.Each(func(docId int, ranges *datastructures.IDSet) { |
| 61 | + ranges.Each(func(rangeId int) { |
| 62 | + // Pagerank source node is doc ID for this reference range. |
| 63 | + refDocId := rangeToDoc[rangeId] |
| 64 | + |
| 65 | + // Pagerank dest nodes are doc IDs for the associated definition range(s). |
| 66 | + // I.e., if a definition is split across files, they each get an edge for now. |
| 67 | + if data, ok := state.DefinitionData[state.RangeData[rangeId].DefinitionResultID]; ok { |
| 68 | + for _, defDocId := range data.UnorderedKeys() { |
| 69 | + // Insert a link for the PageRank calculator. |
| 70 | + if refDocId != defDocId { |
| 71 | + edges[refDocId] = defDocId |
| 72 | + } |
| 73 | + } |
| 74 | + } |
| 75 | + }) |
| 76 | + }) |
| 77 | + } |
| 78 | + return &edges |
| 79 | +} |
| 80 | + |
| 81 | +// Treat implementations as references to the type being implemented. |
| 82 | +// |
| 83 | +// Note: These edges seem to have a dramatic impact on the results. This adds in millions |
| 84 | +// of edges and tends to push up interfaces and classes with lots of implementations. This |
| 85 | +// leads to their effect being a bit overwhelming, compared to when only call/use references |
| 86 | +// are included as pagerank edges. Moreover, some very strange results seem to be bubbling |
| 87 | +// up for Java indexes (about 20% don't look like they should be in the top results). |
| 88 | +// |
| 89 | +// So for now this is disabled by default. |
| 90 | +// TODO(stevey): pagerank.go shouldn't read the flag; it should be passed as a config option. |
| 91 | +func addImplementationEdges(state *conversion.State, edges *map[int]int) { |
| 92 | + graph := *edges |
| 93 | + for _, docMap := range state.ImplementationData { |
| 94 | + docMap.Each(func(docId int, ranges *datastructures.IDSet) { |
| 95 | + // We interpret the destination vertex as the thing being implemented |
| 96 | + // (e.g., the definition of an interface or abstract class). |
| 97 | + destNode := docId |
| 98 | + ranges.Each(func(rangeId int) { |
| 99 | + if data, ok := state.ImplementationData[state.RangeData[rangeId].ImplementationResultID]; ok { |
| 100 | + for _, implDocId := range data.UnorderedKeys() { |
| 101 | + if destNode != implDocId { // skip self-references for pagerank |
| 102 | + graph[implDocId] = destNode |
| 103 | + } |
| 104 | + } |
| 105 | + } |
| 106 | + }) |
| 107 | + }) |
| 108 | + } |
| 109 | +} |
| 110 | + |
| 111 | +// The API to this PageRank package is that you get one shot at seeing the results. |
| 112 | +// Rank the graph, and toss each file/rank pair into the result set. |
| 113 | +func runPageRanker(state *conversion.State, edges *map[int]int) *[]Document { |
| 114 | + graph := pagerank.New() |
| 115 | + |
| 116 | + for srcDocId, targetDocId := range *edges { |
| 117 | + graph.Link(srcDocId, targetDocId) |
| 118 | + } |
| 119 | + |
| 120 | + rankings := make([]Document, 0, len(state.DocumentData)) |
| 121 | + |
| 122 | + graph.Rank(FOLLOW_LINK_CHANCE, TOLERANCE, func(docId int, rank float64) { |
| 123 | + rankings = append(rankings, Document{docId: docId, filePath: state.DocumentData[docId], rank: rank}) |
| 124 | + }) |
| 125 | + return &rankings |
| 126 | +} |
0 commit comments