yhchen
/
ptt-sententree
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

全面改用tfidf

Browse Source
dev/addPush
Zovjsra 4 years ago
parent 4478faa2e7
commit 8237ed570a
10 changed files with 420 additions and 6 deletions
Show Stats Download Patch File Download Diff File

2
app.py
Unescape View File

@ -157,4 +157,4 @@ def dcard_dev():
if __name__ == "__main__":
app.run(debug=False, port=4998, host='0.0.0.0', threaded=True)
app.run(debug=True, port=4998, host='0.0.0.0', threaded=True)

26
static/css/main.css
Unescape View File

@ -70,6 +70,32 @@ html {
box-shadow: 0px 5px 20px rgba(0, 0, 0, .3);
}
#idfEditor {
display: block;
position: fixed;
background-color: #FFF;
left: 50%;
top: 50%;
transform: translate(-50%, -50%);
width: 653px;
height: 600px;
border: lightgray;
border-width: 1px;
border-style: solid;
border-radius: 20px;
padding: 26px;
z-index: 99;
box-shadow: 0px 5px 20px rgba(0, 0, 0, .3);
}
#ieTableContainer {
position: relative;
height: 77%;
overflow: auto;
overflow-y: auto;
transition-duration: 0.5s;
}
#sweContainer {
position: relative;
height: 77%;

102
static/js/dcard.js
Unescape View File

@ -16,7 +16,28 @@ function init() {
})
})
changeData(1)
return
$('#idfEditorLayer').click(function(e) {
if ($('#idfEditorLayer').is(e.target)) {
hideIdfEditor()
}
})
}
function scrollIdfList() {
let targetWord = $('#idfTarget').val()
let wordList = $("#ieContainer").find("tr").slice(1).map((_, a) => { return $($(a).find("td")[0]).html() }).get()
console.log({ "targetWord": targetWord, 'list': wordList })
if (targetWord != '') {
if (wordList.indexOf(targetWord) >= 0) {
} else {
wordList.push(targetWord)
$('#ieContainer').find('tbody').append($('<tr>').attr('class', 'w3-display-container').append($('<td>').attr('style', 'vertical-align: middle;').append(targetWord)).append($('<td>').attr('class', 'w3-right-align').attr('style', 'vertical-align: middle;').append($('<input>').attr('type', 'number').val(Object.keys(idfTable).indexOf(targetWord) < 0 ? 0 : idfTable[targetWord]))))
}
console.log($($('#ieContainer').children('tbody').find('tr')[[wordList.indexOf(targetWord)]]).position())
$("#ieTableContainer").scrollTop($($('#ieContainer').children('tbody').find('tr')[[wordList.indexOf(targetWord)]]).position().top - 50)
}
$('#newStopWord').val('')
}
function changeData(num) {
@ -44,6 +65,85 @@ function changeData(num) {
}
}
function showIdfEditor() {
$(window).unbind('keydown')
$(window).keydown(function(event) {
if (event.keyCode == 13) {
scrollIdfList()
}
})
$('#ieContainer').empty().append(
$('<thead>').append($('<tr>')
.append($('<th>')
.attr('style', 'position: sticky; top: 0; background: white;')
.append('單詞'))
.append($('<th>'))
.append($('<th>').attr('class', 'w3-right-align')
.attr('style', 'position: sticky; top: 0; background: white;')
.append('單詞頻率')
)
)
)
.append($('<tbody>'))
for (word of Object.entries(idfTable).sort((a, b) => { return (b[1] - a[1]) }).map((a) => { return a[0] }).slice(0, 1000)) {
$('#ieContainer').find('tbody')
.append($('<tr>')
.attr('class', 'w3-display-container')
.append($('<td>')
.attr('style', 'vertical-align: middle;')
.append(word))
.append($('<td>')
.attr('class', 'w3-right-align')
.append($('<button>')
.attr('class', 'general-button')
.html('設為最小')
.click(function() {
$(this)
.parent()
.parent()
.find('input').val(0)
})
)
.append($('<button>')
.attr('class', 'general-button')
.html('設為最大')
.click(function() {
$(this)
.parent()
.parent()
.find('input').val(Object.values(idfTableOrig).reduce((a, b) => {
return a > b ? a : b
}))
})
)
.append($('<button>')
.attr('class', 'general-button')
.html('重設')
.click(function() {
var _word = $($(this)
.parent()
.parent()
.find('td')[0]).html()
$(this)
.parent()
.parent()
.find('input').val(idfTableOrig[_word])
})
)
)
.append($('<td>')
.attr('class', 'w3-right-align')
.attr('style', 'vertical-align: middle;')
.append($('<input>')
.attr('type', 'number')
.val(idfTable[word])
)
)
)
}
$('#idfEditorLayer').removeClass('hidden')
}
function destroyCurrentGraph() {
d3.selectAll('#vis').remove()
d3.select('#graph').append('div').attr('id', 'vis')

3
static/js/loadIdfTable.js
Unescape View File

@ -1,8 +1,11 @@
var idfTable
var idfTableOrig
var idfBase = 1000
$.ajax({
url: '/resource/idfTable.json',
async: false,
success: (data) => {
idfTable = data
idfTableOrig = JSON.parse(JSON.stringify(data))
}
})

132
static/js/ptt.js
Unescape View File

@ -71,6 +71,11 @@ function init() {
hideStopWordEditor()
}
})
$('#idfEditorLayer').click(function(e) {
if ($('#idfEditorLayer').is(e.target)) {
hideIdfEditor()
}
})
}
function clearStopWord() {
@ -102,6 +107,23 @@ function addStopWord() {
$('#newStopWord').val('')
}
function scrollIdfList() {
let targetWord = $('#idfTarget').val()
let wordList = $("#ieContainer").find("tr").slice(1).map((_, a) => { return $($(a).find("td")[0]).html() }).get()
console.log({ "targetWord": targetWord, 'list': wordList })
if (targetWord != '') {
if (wordList.indexOf(targetWord) >= 0) {
} else {
wordList.push(targetWord)
$('#ieContainer').find('tbody').append($('<tr>').attr('class', 'w3-display-container').append($('<td>').attr('style', 'vertical-align: middle;').append(targetWord)).append($('<td>').attr('class', 'w3-right-align').attr('style', 'vertical-align: middle;').append($('<input>').attr('type', 'number').val(Object.keys(idfTable).indexOf(targetWord) < 0 ? 0 : idfTable[targetWord]))))
}
console.log($($('#ieContainer').children('tbody').find('tr')[[wordList.indexOf(targetWord)]]).position())
$("#ieTableContainer").scrollTop($($('#ieContainer').children('tbody').find('tr')[[wordList.indexOf(targetWord)]]).position().top - 50)
}
$('#newStopWord').val('')
}
function showStopwordEditor() {
$(window).unbind('keydown')
$(window).keydown(function(event) {
@ -111,7 +133,7 @@ function showStopwordEditor() {
})
$('#sweContainer').empty()
for (word of stopwords) {
$('#sweContainer').append($('<li>').attr('class', 'w3-display-container').append($('<span>').append(word)).append($('<span>').attr('class', 'w3-button w3-hover-red w3-transparent w3-display-right').click(function(e) {
$('#sweContainer').append($('<li>').append($('<span>').append(word)).append($('<span>').attr('class', 'w3-button w3-hover-red w3-transparent w3-display-right').click(function(e) {
var index = $(this).parent().index()
console.log(stopwords[index])
stopwords.splice(index, 1)
@ -122,6 +144,85 @@ function showStopwordEditor() {
$('#stopWordEditorLayer').removeClass('hidden')
}
function showIdfEditor() {
$(window).unbind('keydown')
$(window).keydown(function(event) {
if (event.keyCode == 13) {
scrollIdfList()
}
})
$('#ieContainer').empty().append(
$('<thead>').append($('<tr>')
.append($('<th>')
.attr('style', 'position: sticky; top: 0; background: white;')
.append('單詞'))
.append($('<th>'))
.append($('<th>').attr('class', 'w3-right-align')
.attr('style', 'position: sticky; top: 0; background: white;')
.append('單詞頻率')
)
)
)
.append($('<tbody>'))
for (word of Object.entries(idfTable).sort((a, b) => { return (b[1] - a[1]) }).map((a) => { return a[0] }).slice(0, 1000)) {
$('#ieContainer').find('tbody')
.append($('<tr>')
.attr('class', 'w3-display-container')
.append($('<td>')
.attr('style', 'vertical-align: middle;')
.append(word))
.append($('<td>')
.attr('class', 'w3-right-align')
.append($('<button>')
.attr('class', 'general-button')
.html('設為最小')
.click(function() {
$(this)
.parent()
.parent()
.find('input').val(0)
})
)
.append($('<button>')
.attr('class', 'general-button')
.html('設為最大')
.click(function() {
$(this)
.parent()
.parent()
.find('input').val(Object.values(idfTableOrig).reduce((a, b) => {
return a > b ? a : b
}))
})
)
.append($('<button>')
.attr('class', 'general-button')
.html('重設')
.click(function() {
var _word = $($(this)
.parent()
.parent()
.find('td')[0]).html()
$(this)
.parent()
.parent()
.find('input').val(idfTableOrig[_word])
})
)
)
.append($('<td>')
.attr('class', 'w3-right-align')
.attr('style', 'vertical-align: middle;')
.append($('<input>')
.attr('type', 'number')
.val(idfTable[word])
)
)
)
}
$('#idfEditorLayer').removeClass('hidden')
}
function hideStopWordEditor() {
$(window).unbind('keydown')
$(window).keydown(function(event) {
@ -133,11 +234,40 @@ function hideStopWordEditor() {
$('#stopWordEditorLayer').addClass('hidden')
}
function hideIdfEditor() {
$(window).unbind('keydown')
$(window).keydown(function(event) {
if (event.keyCode == 13) {
event.preventDefault()
sendRequest()
}
})
$('#idfEditorLayer').addClass('hidden')
}
function updateIdfTable() {
let wordList = $("#ieContainer").find("tr").slice(1).map((_, a) => {
return [
[$($(a).find("td")[0]).html(), $($(a).find("td")[1]).find('input').val()]
]
}).get()
console.log(wordList)
for (p of wordList) {
idfTable[p[0]] = p[1]
}
console.log(idfTable)
}
function downloadStopWord() {
stopWordString = stopwords.join('\n')
download(stopWordString, 'stopwords.txt', 'text/plain')
}
function downloadIdfTable() {
let idfTableJson = JSON.stringify(idfTable, null, "\t")
download(idfTableJson, 'idfTable.json', 'text/plain')
}
function hidePopup() {
$('#infoWindowLayer').toggleClass('hidden')

101
static/js/pttPush.js
Unescape View File

@ -58,6 +58,11 @@ function init() {
hideStopWordEditor()
}
})
$('#idfEditorLayer').click(function(e) {
if ($('#idfEditorLayer').is(e.target)) {
hideIdfEditor()
}
})
}
function clearStopWord() {
@ -65,6 +70,23 @@ function clearStopWord() {
$('#sweContainer').html('')
}
function scrollIdfList() {
let targetWord = $('#idfTarget').val()
let wordList = $("#ieContainer").find("tr").slice(1).map((_, a) => { return $($(a).find("td")[0]).html() }).get()
console.log({ "targetWord": targetWord, 'list': wordList })
if (targetWord != '') {
if (wordList.indexOf(targetWord) >= 0) {
} else {
wordList.push(targetWord)
$('#ieContainer').find('tbody').append($('<tr>').attr('class', 'w3-display-container').append($('<td>').attr('style', 'vertical-align: middle;').append(targetWord)).append($('<td>').attr('class', 'w3-right-align').attr('style', 'vertical-align: middle;').append($('<input>').attr('type', 'number').val(Object.keys(idfTable).indexOf(targetWord) < 0 ? 0 : idfTable[targetWord]))))
}
console.log($($('#ieContainer').children('tbody').find('tr')[[wordList.indexOf(targetWord)]]).position())
$("#ieTableContainer").scrollTop($($('#ieContainer').children('tbody').find('tr')[[wordList.indexOf(targetWord)]]).position().top - 50)
}
$('#newStopWord').val('')
}
function addStopWord() {
newswRaw = $('#newStopWord').val()
newswList = newswRaw.split(' ')
@ -89,6 +111,85 @@ function addStopWord() {
$('#newStopWord').val('')
}
function showIdfEditor() {
$(window).unbind('keydown')
$(window).keydown(function(event) {
if (event.keyCode == 13) {
scrollIdfList()
}
})
$('#ieContainer').empty().append(
$('<thead>').append($('<tr>')
.append($('<th>')
.attr('style', 'position: sticky; top: 0; background: white;')
.append('單詞'))
.append($('<th>'))
.append($('<th>').attr('class', 'w3-right-align')
.attr('style', 'position: sticky; top: 0; background: white;')
.append('單詞頻率')
)
)
)
.append($('<tbody>'))
for (word of Object.entries(idfTable).sort((a, b) => { return (b[1] - a[1]) }).map((a) => { return a[0] }).slice(0, 1000)) {
$('#ieContainer').find('tbody')
.append($('<tr>')
.attr('class', 'w3-display-container')
.append($('<td>')
.attr('style', 'vertical-align: middle;')
.append(word))
.append($('<td>')
.attr('class', 'w3-right-align')
.append($('<button>')
.attr('class', 'general-button')
.html('設為最小')
.click(function() {
$(this)
.parent()
.parent()
.find('input').val(0)
})
)
.append($('<button>')
.attr('class', 'general-button')
.html('設為最大')
.click(function() {
$(this)
.parent()
.parent()
.find('input').val(Object.values(idfTableOrig).reduce((a, b) => {
return a > b ? a : b
}))
})
)
.append($('<button>')
.attr('class', 'general-button')
.html('重設')
.click(function() {
var _word = $($(this)
.parent()
.parent()
.find('td')[0]).html()
$(this)
.parent()
.parent()
.find('input').val(idfTableOrig[_word])
})
)
)
.append($('<td>')
.attr('class', 'w3-right-align')
.attr('style', 'vertical-align: middle;')
.append($('<input>')
.attr('type', 'number')
.val(idfTable[word])
)
)
)
}
$('#idfEditorLayer').removeClass('hidden')
}
function showStopwordEditor() {
console.log(stopwords)
$(window).unbind('keydown')

7
static/node_modules/sententree/dist/sententree-standalone.js generated vendored
Unescape View File

@ -19143,9 +19143,12 @@
words.forEach(function(w) {
var value = fdist[w];
if (value < maxSupport && (value / Math.max(idfTable[itemset[w]], 1000)) * 1000 > maxc) {
//if (value < maxSupport && (value / Math.max(idfTable[itemset[w]], idfBase)) * idfBase > maxc) {
//if (value < maxSupport && value > maxc && (stopwords.indexOf(itemset[w]) < 0 || isNotRoot)) {
if (value < maxSupport && value > maxc && stopwords.indexOf(itemset[w]) < 0) {
maxw = +w;
maxc = (value / Math.max(idfTable[itemset[w]], 1000)) * 1000;
//maxc = (value / Math.max(idfTable[itemset[w]], idfBase)) * idfBase
maxc = value
}
});
console.log(itemset)

17
templates/dcard.html
Unescape View File

@ -36,6 +36,23 @@
</div>
</div>
</div>
<div id="idfEditorLayer" class="info hidden">
<div id="idfEditor">
<h4 id="ieTitle" style="margin:10px; display: inline;">編輯停用詞</h4>
<div id='ieTableContainer'>
<table id="ieContainer" class="w3-table w3-bordered w3-hoverable" style="margin-bottom: 10px;"></table>
</div>
<div>
<input class="w3-input w3-border" style="width: 85%; display: inline;" type="text" id="idfTarget" placeholder="輸入單詞">
<button class="general-button w3-right" type="button" id="confirm" style="background-color: #379; margin-left: 8px;" onclick="scrollIdfList()">移動</button>
</div>
<div id="ieButtons" style="margin: 20px 0px;">
<button class="general-button" type="button" id="confirm" style="background-color: #379; margin: 0px 10px" onclick="updateIdfTable(); hideIdfEditor(); destroyCurrentGraph(); buildSentetree()">確認</button>
<button class="general-button" type="button" id="confirm" style="background-color: #379; margin: 0px 10px" onclick="downloadIdfTable()">匯出頻率表</button>
<button class="general-button w3-right" type="button" id="confirm" style="background-color: #379; margin: 0px 20px" onclick="hideIdfEditor()">返回</button>
</div>
</div>
</div>
<div id="titleListLayer" class="info hidden">
<div id="titleList">
<h2 id="titleListKeyword" style="margin:10px; display: inline;"></h2>

19
templates/ptt.html
Unescape View File

@ -34,6 +34,23 @@
</div>
</div>
</div>
<div id="idfEditorLayer" class="info hidden">
<div id="idfEditor">
<h4 id="ieTitle" style="margin:10px; display: inline;">編輯停用詞</h4>
<div id='ieTableContainer'>
<table id="ieContainer" class="w3-table w3-bordered w3-hoverable" style="margin-bottom: 10px;"></table>
</div>
<div>
<input class="w3-input w3-border" style="width: 85%; display: inline;" type="text" id="idfTarget" placeholder="輸入單詞">
<button class="general-button w3-right" type="button" id="confirm" style="background-color: #379; margin-left: 8px;" onclick="scrollIdfList()">移動</button>
</div>
<div id="ieButtons" style="margin: 20px 0px;">
<button class="general-button" type="button" id="confirm" style="background-color: #379; margin: 0px 10px" onclick="updateIdfTable(); hideIdfEditor(); destroyCurrentGraph(); buildSentetree()">確認</button>
<button class="general-button" type="button" id="confirm" style="background-color: #379; margin: 0px 10px" onclick="downloadIdfTable()">匯出頻率表</button>
<button class="general-button w3-right" type="button" id="confirm" style="background-color: #379; margin: 0px 20px" onclick="hideIdfEditor()">返回</button>
</div>
</div>
</div>
<div id="titleListLayer" class="info hidden">
<div id="titleList">
<h2 id="titleListKeyword" style="margin:10px; display: inline;"></h2>
@ -63,7 +80,7 @@
<input id="keywordBox" type="text" name="message" placeholder="輸入關鍵詞">
<button class="general-button" type="button" id="sendButton" onclick="sendRequest()">搜尋關鍵字</button>
<button class="general-button" type="button" id="resetButton" onclick="document.getElementById('keywordBox').value=''">清除關鍵字</button>
<button class="general-button" type="button" id="editSWButton" onclick="showStopwordEditor()">編輯停用詞</button>
<button class="general-button" type="button" id="editSWButton" onclick="showIdfEditor()">編輯停用詞</button>
</form>
</div>
<div id="advancedArea">

17
templates/ptt_push.html
Unescape View File

@ -35,6 +35,23 @@
</div>
</div>
</div>
<div id="idfEditorLayer" class="info hidden">
<div id="idfEditor">
<h4 id="ieTitle" style="margin:10px; display: inline;">編輯停用詞</h4>
<div id='ieTableContainer'>
<table id="ieContainer" class="w3-table w3-bordered w3-hoverable" style="margin-bottom: 10px;"></table>
</div>
<div>
<input class="w3-input w3-border" style="width: 85%; display: inline;" type="text" id="idfTarget" placeholder="輸入單詞">
<button class="general-button w3-right" type="button" id="confirm" style="background-color: #379; margin-left: 8px;" onclick="scrollIdfList()">移動</button>
</div>
<div id="ieButtons" style="margin: 20px 0px;">
<button class="general-button" type="button" id="confirm" style="background-color: #379; margin: 0px 10px" onclick="updateIdfTable(); hideIdfEditor(); destroyCurrentGraph(); buildSentetree()">確認</button>
<button class="general-button" type="button" id="confirm" style="background-color: #379; margin: 0px 10px" onclick="downloadIdfTable()">匯出頻率表</button>
<button class="general-button w3-right" type="button" id="confirm" style="background-color: #379; margin: 0px 20px" onclick="hideIdfEditor()">返回</button>
</div>
</div>
</div>
<div id="titleListLayer" class="info hidden">
<div id="titleList">
<h2 id="titleListKeyword" style="margin:10px; display: inline;"></h2>

Write Preview
Loading…
Cancel
Save