424 lines
16 KiB
Swift
424 lines
16 KiB
Swift
//
|
|
// KissConsole+DB.swift
|
|
// KissMeConsole
|
|
//
|
|
// Created by ened-book-m1 on 11/3/24.
|
|
//
|
|
|
|
import Foundation
|
|
import KissMe
|
|
import KissMeme
|
|
|
|
|
|
enum CandleDataFieldType: UInt8 {
|
|
case uint8 = 1 // 8 bits unsigned integer
|
|
case uint16 = 2 // 16 bits unsigned integer
|
|
case uint32 = 4 // 32 bits unsigned integer
|
|
case uint64 = 8 // 64 bits unsigned integer
|
|
case double = 10 // 8 byte float point
|
|
case float = 11 // 4 byte float point
|
|
}
|
|
|
|
extension CandleDataFieldType: CustomStringConvertible {
|
|
var description: String {
|
|
switch self {
|
|
case .uint8: return "uint8"
|
|
case .uint16: return "uint16"
|
|
case .uint32: return "uint32"
|
|
case .uint64: return "uint64"
|
|
case .double: return "double"
|
|
case .float: return "float"
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
extension Int64 {
|
|
var fieldType: CandleDataFieldType {
|
|
let unsignedValue = UInt64(bitPattern: self)
|
|
if unsignedValue & ~UInt64(UInt8.max) == 0 {
|
|
return .uint8
|
|
}
|
|
else if unsignedValue & ~UInt64(UInt16.max) == 0 {
|
|
return .uint16
|
|
}
|
|
else if unsignedValue & ~UInt64(UInt32.max) == 0 {
|
|
return .uint32
|
|
}
|
|
else if unsignedValue & ~UInt64.max == 0 {
|
|
return .uint64
|
|
}
|
|
// If the value cannot be represented as an unsigned integer, check for float or double
|
|
else {
|
|
// Check if the value can be represented as a Float
|
|
if self >= Int64(Float.leastNonzeroMagnitude.bitPattern) && self <= Int64(Float.greatestFiniteMagnitude.bitPattern) {
|
|
return .float
|
|
}
|
|
// Otherwise, use Double
|
|
else {
|
|
return .double
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
extension Domestic.Candle: @retroactive Equatable {
|
|
|
|
var keyOfCandleData: Data? {
|
|
guard let date = stockFullDate.yyyyMMddHHmmss_UTC_toDate else {
|
|
return nil
|
|
}
|
|
return Data(value: UInt32(date.timeIntervalSince2020))
|
|
}
|
|
|
|
var isValidInMarketTime: Bool {
|
|
guard let (hh, mm, ss) = stockConclusionTime.HHmmss else {
|
|
return false
|
|
}
|
|
let timeInSeconds = (hh * 3600 + mm * 60 + ss)
|
|
return 9 * 3600 <= timeInSeconds && timeInSeconds <= 18 * 3600
|
|
}
|
|
|
|
public static func == (lhs: Domestic.Candle, rhs: Domestic.Candle) -> Bool {
|
|
return
|
|
lhs.stockBusinessDate == rhs.stockBusinessDate &&
|
|
lhs.stockConclusionTime == rhs.stockConclusionTime &&
|
|
lhs.accumulatedTradingAmount == rhs.accumulatedTradingAmount &&
|
|
lhs.currentStockPrice == rhs.currentStockPrice &&
|
|
lhs.stockOpenningPrice == rhs.stockOpenningPrice &&
|
|
lhs.highestStockPrice == rhs.highestStockPrice &&
|
|
lhs.lowestStockPrice == rhs.lowestStockPrice &&
|
|
lhs.conclusionVolume == rhs.conclusionVolume
|
|
}
|
|
}
|
|
|
|
|
|
struct CandleData {
|
|
let key: Data
|
|
let data: Data
|
|
|
|
var candleKey: UInt32 { key.value_UInt32 }
|
|
var candleDate: String { Date(timeIntervalSince2020: TimeInterval(key.value_UInt32)).yyyyMMddHHmmss_UTC }
|
|
|
|
init(key: Data, data: Data) {
|
|
self.key = key
|
|
self.data = data
|
|
}
|
|
|
|
init(candle: Domestic.Candle) throws {
|
|
guard let keyOfCandleData = candle.keyOfCandleData else {
|
|
throw GeneralError.invalidCandleValue(candle.stockFullDate + ": invalid")
|
|
}
|
|
self.key = keyOfCandleData
|
|
|
|
guard let accumulatedTradingAmount = Int64(candle.accumulatedTradingAmount) else {
|
|
throw GeneralError.invalidCandleValue(candle.stockFullDate + ": accumulatedTradingAmount = \(candle.accumulatedTradingAmount)")
|
|
}
|
|
guard let currentStockPrice = Int64(candle.currentStockPrice) else {
|
|
throw GeneralError.invalidCandleValue(candle.stockFullDate + ": currentStockPrice = \(candle.currentStockPrice)")
|
|
}
|
|
guard let stockOpenningPrice = Int64(candle.stockOpenningPrice) else {
|
|
throw GeneralError.invalidCandleValue(candle.stockFullDate + ": stockOpenningPrice = \(candle.stockOpenningPrice)")
|
|
}
|
|
guard let highestStockPrice = Int64(candle.highestStockPrice) else {
|
|
throw GeneralError.invalidCandleValue(candle.stockFullDate + ": highestStockPrice = \(candle.highestStockPrice)")
|
|
}
|
|
guard let lowestStockPrice = Int64(candle.lowestStockPrice) else {
|
|
throw GeneralError.invalidCandleValue(candle.stockFullDate + ": lowestStockPrice = \(candle.lowestStockPrice)")
|
|
}
|
|
guard let conclusionVolume = Int64(candle.conclusionVolume) else {
|
|
throw GeneralError.invalidCandleValue(candle.stockFullDate + ": conclusionVolume = \(candle.conclusionVolume)")
|
|
}
|
|
|
|
let values = [accumulatedTradingAmount, currentStockPrice, stockOpenningPrice, highestStockPrice, lowestStockPrice, conclusionVolume]
|
|
|
|
var typeFields = [UInt8]()
|
|
var valuesData = Data()
|
|
for value in values {
|
|
let valueData: Data
|
|
|
|
let fieldType = value.fieldType
|
|
typeFields.append(fieldType.rawValue)
|
|
|
|
switch fieldType {
|
|
case .uint8: valueData = Data(value: UInt8(value))
|
|
case .uint16: valueData = Data(value: UInt16(value))
|
|
case .uint32: valueData = Data(value: UInt32(value))
|
|
case .uint64: valueData = Data(value: UInt64(value))
|
|
case .float: valueData = Data(value: Float(value))
|
|
case .double: valueData = Data(value: Double(value))
|
|
}
|
|
valuesData.append(valueData)
|
|
}
|
|
|
|
var data = Data()
|
|
data.append(contentsOf: typeFields)
|
|
data.append(valuesData)
|
|
self.data = data
|
|
//print("data: \(data.count)")
|
|
}
|
|
|
|
var candle: Domestic.Candle {
|
|
let stockFullDate = Date(timeIntervalSince2020: TimeInterval(key.value_UInt32)).yyyyMMddHHmmss_UTC
|
|
assert(stockFullDate.count == 8+6, "invalid key length")
|
|
let stockBusinessDate = String(stockFullDate.prefix(8))
|
|
let stockConclusionTime = String(stockFullDate.suffix(6))
|
|
|
|
let typeFields = [UInt8](data[0 ..< 6])
|
|
var values = [stockBusinessDate, stockConclusionTime]
|
|
|
|
//print("candle data: \(data.count)")
|
|
|
|
var start = 6
|
|
for field in typeFields {
|
|
let value: String
|
|
|
|
switch CandleDataFieldType(rawValue: field)! {
|
|
case .uint8:
|
|
value = String(data.subdata(in: start ..< start+1).value_UInt8)
|
|
start += 1
|
|
|
|
case .uint16:
|
|
value = String(data.subdata(in: start ..< start+2).value_UInt16)
|
|
start += 2
|
|
|
|
case .uint32:
|
|
value = String(data.subdata(in: start ..< start+4).value_UInt32)
|
|
start += 4
|
|
|
|
case .uint64:
|
|
value = String(data.subdata(in: start ..< start+8).value_UInt64)
|
|
start += 8
|
|
|
|
case .float:
|
|
value = String(data.subdata(in: start ..< start+4).value_Float)
|
|
start += 4
|
|
|
|
case .double:
|
|
value = String(data.subdata(in: start ..< start+8).value_Double)
|
|
start += 8
|
|
}
|
|
values.append(value)
|
|
}
|
|
return try! Domestic.Candle(array: values, source: "")
|
|
}
|
|
}
|
|
|
|
|
|
class CandleMinuteFileName {
|
|
let regex: NSRegularExpression
|
|
|
|
init() {
|
|
let pattern = ".*/(\\d{6})/min/candle-(\\d{8})\\.csv$"
|
|
regex = try! NSRegularExpression(pattern: pattern, options: [])
|
|
}
|
|
|
|
func matchedUrl(_ fileUrl: String) -> (productNo: String, yyyyMMdd: String)? {
|
|
let range = NSRange(location: 0, length: fileUrl.utf16.count)
|
|
let results = regex.matches(in: fileUrl, range: range)
|
|
let fragments = results.map { result in
|
|
(0 ..< result.numberOfRanges).map {
|
|
let nsRange = result.range(at: $0)
|
|
if let range = Range(nsRange, in: fileUrl) {
|
|
return String(fileUrl[range])
|
|
}
|
|
return ""
|
|
}
|
|
}
|
|
if let first = fragments.first, first.count == 3 {
|
|
return (first[1], first[2])
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
|
|
|
|
extension KissConsole {
|
|
|
|
private func subPathForProduct(productNo: String?) -> FileManager.DirectoryEnumerator? {
|
|
if let productNo = productNo {
|
|
return FileManager.subPathFiles("data/\(productNo)")
|
|
}
|
|
else {
|
|
return FileManager.subPathFiles("data")
|
|
}
|
|
}
|
|
|
|
func collectCandleMinuteFiles(productNo: String?, year: String?, month: String?, day: String?) -> [String: [URL]] {
|
|
print("Collecting candle csv for \(productNo ?? "all") at \(year ?? "")\(month ?? "")\(day ?? "") .....")
|
|
guard let enumerator = subPathForProduct(productNo: productNo) else {
|
|
return [:]
|
|
}
|
|
|
|
let candleMinName = CandleMinuteFileName()
|
|
var candleFiles = [String: [URL]]()
|
|
for case let fileUrl as URL in enumerator {
|
|
guard let (fileProductNo, yyyyMMdd) = candleMinName.matchedUrl(fileUrl.path) else {
|
|
continue
|
|
}
|
|
if let productNo = productNo, productNo != fileProductNo {
|
|
continue
|
|
}
|
|
if let year = year, yyyyMMdd.prefix(4) != year {
|
|
continue
|
|
}
|
|
if let month = month, yyyyMMdd.dropFirst(4).prefix(2) != month {
|
|
continue
|
|
}
|
|
if let day = day, yyyyMMdd.suffix(2) != day {
|
|
continue
|
|
}
|
|
|
|
if candleFiles.keys.contains(fileProductNo) {
|
|
candleFiles[fileProductNo]!.append(fileUrl)
|
|
}
|
|
else {
|
|
candleFiles[fileProductNo] = [fileUrl]
|
|
}
|
|
}
|
|
return candleFiles
|
|
}
|
|
|
|
/// - Parameters:
|
|
/// remoeOldDB: 이미 생성된 DB 파일이 있으면 삭제함.
|
|
/// trimAfterMarket: 거래시간 09:00 ~ 16:00 을 벗어난 경우, DB 에 기록하지 않음.
|
|
func buildCandleMinuteDB(productNo: String, csvFiles: [URL], removeOldDB: Bool = false, trimAfterMarket: Bool = true) -> Bool {
|
|
for csvFile in csvFiles {
|
|
let candleMinName = CandleMinuteFileName()
|
|
if let (_, yyyyMMdd) = candleMinName.matchedUrl(csvFile.path), let year = Int(yyyyMMdd.prefix(4)) {
|
|
let yearDbPath = Self.candleMinuteDBPath(productNo: productNo, year: String(year))
|
|
if removeOldDB {
|
|
try? FileManager.default.removeItem(at: yearDbPath)
|
|
}
|
|
try? FileManager.default.createDirectory(at: yearDbPath, withIntermediateDirectories: true)
|
|
|
|
do {
|
|
let candles = try [Domestic.Candle].readCsv(fromFile: csvFile)
|
|
|
|
let db = try KissDB(directory: yearDbPath)
|
|
try db.begin()
|
|
|
|
for candle in candles {
|
|
if trimAfterMarket {
|
|
guard candle.isValidInMarketTime else {
|
|
continue
|
|
}
|
|
}
|
|
let candleData = try CandleData(candle: candle)
|
|
let item = KissDB.DataItem(key: candleData.key, value: candleData.data)
|
|
try db.insertData(item: item)
|
|
|
|
if candleData.candle != candle {
|
|
assertionFailure("invalid candle data")
|
|
}
|
|
}
|
|
|
|
try db.commit()
|
|
} catch {
|
|
print("\(error)")
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
static func candleMinuteDBPath(productNo: String, year: String) -> URL {
|
|
let dataDbPath = URL.currentDirectory().appending(path: "data-db")
|
|
let yearDbPath = dataDbPath.appending(path: "\(productNo)/min/candle-\(year).db1")
|
|
return yearDbPath
|
|
}
|
|
|
|
/// - Parameters:
|
|
/// trimAfterMarket: 거래시간 09:00 ~ 16:00 을 벗어난 경우, DB 의 검사에서 제외함.
|
|
func validateCandleMinuteDB(productNo: String, year: String, trimAfterMarket: Bool = true) -> Bool {
|
|
let yearDbPath = Self.candleMinuteDBPath(productNo: productNo, year: year)
|
|
guard let (_, yearCsvFiles) = collectCandleMinuteFiles(productNo: productNo, year: year, month: nil, day: nil).first else {
|
|
print("No csv files of productNo: \(productNo), year: \(year)")
|
|
return false
|
|
}
|
|
do {
|
|
let db = try KissDB(directory: yearDbPath)
|
|
|
|
try db.begin()
|
|
let totalYearItems = db.count
|
|
print("Total candle db items: \(totalYearItems), productNo: \(productNo), year: \(year)")
|
|
|
|
var itemCount = 0
|
|
for csvFile in yearCsvFiles {
|
|
let candles = try [Domestic.Candle].readCsv(fromFile: csvFile)
|
|
for candle in candles {
|
|
if trimAfterMarket {
|
|
guard candle.isValidInMarketTime else {
|
|
continue
|
|
}
|
|
}
|
|
guard let keyOfCandleData = candle.keyOfCandleData else {
|
|
continue
|
|
}
|
|
try db.selectData(key: keyOfCandleData) { dataItem in
|
|
itemCount += 1
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
|
|
print("Total candle CSV items: \(itemCount)")
|
|
if totalYearItems != itemCount {
|
|
print("Error of invalid candle db items!!")
|
|
return false
|
|
}
|
|
} catch {
|
|
print("\(error)")
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func countCandleMinunteDB(productNo: String, year: String) -> Int {
|
|
let yearDbPath = Self.candleMinuteDBPath(productNo: productNo, year: year)
|
|
do {
|
|
let db = try KissDB(directory: yearDbPath)
|
|
try db.begin()
|
|
let count = Int(db.count)
|
|
try db.rollback()
|
|
return count
|
|
} catch {
|
|
print("\(error)")
|
|
return 0
|
|
}
|
|
}
|
|
|
|
func selectCandleMinuteDB(productNo: String, year: Int, month: Int, day: Int, hour: Int?) -> [Domestic.Candle] {
|
|
let requestCandleDate: String
|
|
if let hour = hour {
|
|
requestCandleDate = String(format: "%04d%02d%02d%02d", year, month, day, hour)
|
|
}
|
|
else {
|
|
requestCandleDate = String(format: "%04d%02d%02d", year, month, day)
|
|
}
|
|
|
|
var candleItems = [Domestic.Candle]()
|
|
let yearDbPath = Self.candleMinuteDBPath(productNo: productNo, year: String(year))
|
|
do {
|
|
let db = try KissDB(directory: yearDbPath)
|
|
|
|
try db.begin()
|
|
|
|
try db.selectData { dataItem in
|
|
let candleData = CandleData(key: dataItem.key, data: dataItem.value)
|
|
if candleData.candleDate.prefix(requestCandleDate.count) == requestCandleDate {
|
|
candleItems.append(candleData.candle)
|
|
}
|
|
return true
|
|
}
|
|
|
|
try db.rollback()
|
|
} catch {
|
|
print("\(error)")
|
|
}
|
|
return candleItems
|
|
}
|
|
}
|