Add KissDB from CandleData

This commit is contained in:
2024-11-08 01:14:20 +09:00
parent 4e0c01fed2
commit 9646dbe556
8 changed files with 499 additions and 0 deletions

View File

@@ -48,6 +48,13 @@ extension Date {
return dateFormatter.string(from: self)
}
public var yyyyMMddHHmmss_UTC: String {
let dateFormatter = DateFormatter()
dateFormatter.timeZone = TimeZone(abbreviation: "UTC")
dateFormatter.dateFormat = "yyyyMMddHHmmss"
return dateFormatter.string(from: self)
}
public static var appTime: TimeInterval {
ProcessInfo.processInfo.systemUptime
}
@@ -123,6 +130,134 @@ extension Date {
self = newDate
}
}
private static let timestampAt_20200101_000000: TimeInterval = 1577804400
private static let kstOffset: TimeInterval = {
let kstTimeZone = TimeZone(abbreviation: "KST")!
let utcDate = Date(timeIntervalSince1970: timestampAt_20200101_000000)
return TimeInterval(kstTimeZone.secondsFromGMT(for: utcDate))
}()
public var timeIntervalSince2020: TimeInterval {
let kstDate = Date(timeIntervalSince1970: Self.timestampAt_20200101_000000)
return timeIntervalSince1970 - kstDate.timeIntervalSince1970
}
public init(timeIntervalSince2020 interval: TimeInterval) {
self.init(timeIntervalSince1970: interval + Self.timestampAt_20200101_000000)
}
}
extension Data {
public init(value: UInt8) {
var bigEndianValue = value.bigEndian
self.init(bytes: &bigEndianValue, count: MemoryLayout<UInt8>.size)
}
public init(value: UInt16) {
var bigEndianValue = value.bigEndian
self.init(bytes: &bigEndianValue, count: MemoryLayout<UInt16>.size)
}
public init(value: UInt32) {
var bigEndianValue = value.bigEndian
self.init(bytes: &bigEndianValue, count: MemoryLayout<UInt32>.size)
}
public init(value: UInt64) {
var bigEndianValue = value.bigEndian
self.init(bytes: &bigEndianValue, count: MemoryLayout<UInt64>.size)
}
public init(value: Int64) {
var bigEndianValue = value.bigEndian
self.init(bytes: &bigEndianValue, count: MemoryLayout<Int64>.size)
}
public init(value: Float) {
var bigEndianValue = value.bitPattern.bigEndian
self.init(bytes: &bigEndianValue, count: MemoryLayout<Float>.size)
}
public init(value: Double) {
var bigEndianValue = value.bitPattern.bigEndian
self.init(bytes: &bigEndianValue, count: MemoryLayout<Double>.size)
}
public var value_UInt8: UInt8 {
assert(count == MemoryLayout<UInt8>.size, "invalid key data size")
return withUnsafeBytes { bytes in
guard let baseAddress = bytes.baseAddress else {
fatalError("Invalid base address")
}
return baseAddress.load(as: UInt8.self).bigEndian
}
}
public var value_UInt16: UInt16 {
assert(count == MemoryLayout<UInt16>.size, "invalid key data size")
return withUnsafeBytes { bytes in
guard let baseAddress = bytes.baseAddress else {
fatalError("Invalid base address")
}
return baseAddress.load(as: UInt16.self).bigEndian
}
}
public var value_UInt32: UInt32 {
assert(count == MemoryLayout<UInt32>.size, "invalid key data size")
return withUnsafeBytes { bytes in
guard let baseAddress = bytes.baseAddress else {
fatalError("Invalid base address")
}
return baseAddress.load(as: UInt32.self).bigEndian
}
}
public var value_UInt64: UInt64 {
assert(count == MemoryLayout<UInt64>.size, "invalid key data size")
return withUnsafeBytes { bytes in
guard let baseAddress = bytes.baseAddress else {
fatalError("Invalid base address")
}
return baseAddress.load(as: UInt64.self).bigEndian
}
}
public var value_Int64: Int64 {
assert(count == MemoryLayout<Int64>.size, "invalid key data size")
return withUnsafeBytes { bytes in
guard let baseAddress = bytes.baseAddress else {
fatalError("Invalid base address")
}
return baseAddress.load(as: Int64.self).bigEndian
}
}
public var value_Float: Float {
assert(count == MemoryLayout<Float>.size, "invalid key data size")
return withUnsafeBytes { bytes in
guard let baseAddress = bytes.baseAddress else {
fatalError("Invalid base address")
}
return Float(bitPattern: baseAddress.load(as: UInt32.self).bigEndian)
}
}
public var value_Double: Double {
assert(count == MemoryLayout<Double>.size, "invalid key data size")
return withUnsafeBytes { bytes in
guard let baseAddress = bytes.baseAddress else {
fatalError("Invalid base address")
}
return Double(bitPattern: baseAddress.load(as: UInt64.self).bigEndian)
}
}
public var hexString: String {
return map { String(format: "%02x", $0) }.joined()
}
}
@@ -153,6 +288,18 @@ extension String {
return date
}
public var yyyyMMddHHmmss_UTC_toDate: Date? {
let dateFormatter = DateFormatter()
dateFormatter.dateFormat = "yyyyMMddHHmmss"
dateFormatter.timeZone = TimeZone(secondsFromGMT: 0)
guard let date = dateFormatter.date(from: self) else {
return nil
}
return date
}
public var HHmmss: (Int, Int, Int)? {
guard utf8.count == 6 else {
return nil

View File

@@ -25,6 +25,7 @@ public enum GeneralError: Error {
case invalidCandleCsvFile(String)
case incorrectCsvHeaderField(String)
case noData
case invalidCandleValue(String)
// MARK: WebSocket
case cannotIssueApprovalKey

View File

@@ -1097,6 +1097,7 @@ public struct MinutePriceResult: Codable {
case conclusionVolume = "cntg_vol"
}
/// yyyyMMddHHmmss
public var stockFullDate: String {
return stockBusinessDate + stockConclusionTime
}

View File

@@ -18,6 +18,7 @@ let package = Package(
// Dependencies declare other packages that this package depends on.
//.package(url: "../KissMe", from: "1.0.0"),
.package(path: "../KissMe"),
.package(path: "../libraries/KissMeme/KissMeme"),
],
targets: [
// Targets are the basic building blocks of a package. A target can define a module or a test suite.

View File

@@ -0,0 +1,335 @@
//
// KissConsole+DB.swift
// KissMeConsole
//
// Created by ened-book-m1 on 11/3/24.
//
import Foundation
import KissMe
import KissMeme
extension KissConsole {
}
func test_build_min_db() {
// guard let enumerator = FileManager.subPathFiles("data") else {
// return
// }
//let db = try KissDB(directory: url)
//test_check_name_parsed()
//test_date_time()
//build_min_db_from_candle_csv()
test_select_min_db()
}
private func test_field_type() {
let v1: Int64 = 0
let v2: Int64 = 1
let v3: Int64 = 65536
let v4: Int64 = -2
print("\(v1.fieldType)")
print("\(v2.fieldType)")
print("\(v3.fieldType)")
print("\(v4.fieldType)")
print("done")
}
private func test_date_time() {
let kissDate = Date.date(yyyyMMdd: "20200101", HHmmss: "000000")
let timestamp = UInt64(kissDate!.timeIntervalSince1970)
print("timestamp: \(timestamp)")
print("kissDate: \(kissDate!.timeIntervalSince2020)")
print("today: \(UInt32(Date().timeIntervalSince2020))")
let value: UInt64 = 1234567890
let d = Data(value: value)
print(d.hexString)
}
enum CandleDataFieldType: UInt8 {
case uint8 = 1 // 8 bits unsigned integer
case uint16 = 2 // 16 bits unsigned integer
case uint32 = 4 // 32 bits unsigned integer
case uint64 = 8 // 64 bits unsigned integer
case double = 10 // 8 byte float point
case float = 11 // 4 byte float point
}
extension Int64 {
var fieldType: CandleDataFieldType {
let unsignedValue = UInt64(bitPattern: self)
if unsignedValue & ~UInt64(UInt8.max) == 0 {
return .uint8
}
else if unsignedValue & ~UInt64(UInt16.max) == 0 {
return .uint16
}
else if unsignedValue & ~UInt64(UInt32.max) == 0 {
return .uint32
}
else if unsignedValue & ~UInt64.max == 0 {
return .uint64
}
// If the value cannot be represented as an unsigned integer, check for float or double
else {
// Check if the value can be represented as a Float
if self >= Int64(Float.leastNonzeroMagnitude.bitPattern) && self <= Int64(Float.greatestFiniteMagnitude.bitPattern) {
return .float
}
// Otherwise, use Double
else {
return .double
}
}
}
}
extension Domestic.Candle: @retroactive Equatable {
public static func == (lhs: Domestic.Candle, rhs: Domestic.Candle) -> Bool {
return
lhs.stockBusinessDate == rhs.stockBusinessDate &&
lhs.stockConclusionTime == rhs.stockConclusionTime &&
lhs.accumulatedTradingAmount == rhs.accumulatedTradingAmount &&
lhs.currentStockPrice == rhs.currentStockPrice &&
lhs.stockOpenningPrice == rhs.stockOpenningPrice &&
lhs.highestStockPrice == rhs.highestStockPrice &&
lhs.lowestStockPrice == rhs.lowestStockPrice &&
lhs.conclusionVolume == rhs.conclusionVolume
}
}
struct CandleData {
let key: Data
let data: Data
var candleKey: UInt32 { key.value_UInt32 }
var candleDate: String { Date(timeIntervalSince2020: TimeInterval(key.value_UInt32)).yyyyMMddHHmmss_UTC }
init(key: Data, data: Data) {
self.key = key
self.data = data
}
init(candle: Domestic.Candle) throws {
guard let keyDate = candle.stockFullDate.yyyyMMddHHmmss_UTC_toDate else {
throw GeneralError.invalidCandleValue(candle.stockFullDate + ": stockFullDate = \(candle.stockFullDate)")
}
self.key = Data(value: UInt32(keyDate.timeIntervalSince2020))
guard let accumulatedTradingAmount = Int64(candle.accumulatedTradingAmount) else {
throw GeneralError.invalidCandleValue(candle.stockFullDate + ": accumulatedTradingAmount = \(candle.accumulatedTradingAmount)")
}
guard let currentStockPrice = Int64(candle.currentStockPrice) else {
throw GeneralError.invalidCandleValue(candle.stockFullDate + ": currentStockPrice = \(candle.currentStockPrice)")
}
guard let stockOpenningPrice = Int64(candle.stockOpenningPrice) else {
throw GeneralError.invalidCandleValue(candle.stockFullDate + ": stockOpenningPrice = \(candle.stockOpenningPrice)")
}
guard let highestStockPrice = Int64(candle.highestStockPrice) else {
throw GeneralError.invalidCandleValue(candle.stockFullDate + ": highestStockPrice = \(candle.highestStockPrice)")
}
guard let lowestStockPrice = Int64(candle.lowestStockPrice) else {
throw GeneralError.invalidCandleValue(candle.stockFullDate + ": lowestStockPrice = \(candle.lowestStockPrice)")
}
guard let conclusionVolume = Int64(candle.conclusionVolume) else {
throw GeneralError.invalidCandleValue(candle.stockFullDate + ": conclusionVolume = \(candle.conclusionVolume)")
}
let values = [accumulatedTradingAmount, currentStockPrice, stockOpenningPrice, highestStockPrice, lowestStockPrice, conclusionVolume]
var typeFields = [UInt8]()
var valuesData = Data()
for value in values {
let valueData: Data
let fieldType = value.fieldType
typeFields.append(fieldType.rawValue)
switch fieldType {
case .uint8: valueData = Data(value: UInt8(value))
case .uint16: valueData = Data(value: UInt16(value))
case .uint32: valueData = Data(value: UInt32(value))
case .uint64: valueData = Data(value: UInt64(value))
case .float: valueData = Data(value: Float(value))
case .double: valueData = Data(value: Double(value))
}
valuesData.append(valueData)
}
var data = Data()
data.append(contentsOf: typeFields)
data.append(valuesData)
self.data = data
print("data: \(data.count)")
}
var candle: Domestic.Candle {
let stockFullDate = Date(timeIntervalSince2020: TimeInterval(key.value_UInt32)).yyyyMMddHHmmss_UTC
assert(stockFullDate.count == 8+6, "invalid key length")
let stockBusinessDate = String(stockFullDate.prefix(8))
let stockConclusionTime = String(stockFullDate.suffix(6))
let typeFields = [UInt8](data[0 ..< 6])
var values = [stockBusinessDate, stockConclusionTime]
print("candle data: \(data.count)")
var start = 6
for field in typeFields {
let value: String
switch CandleDataFieldType(rawValue: field)! {
case .uint8: value = String(data.subdata(in: start ..< start+1).value_UInt8); start += 1
case .uint16: value = String(data.subdata(in: start ..< start+2).value_UInt16); start += 2
case .uint32: value = String(data.subdata(in: start ..< start+4).value_UInt32); start += 4
case .uint64: value = String(data.subdata(in: start ..< start+8).value_UInt64); start += 8
case .float: value = String(data.subdata(in: start ..< start+4).value_Float); start += 4
case .double: value = String(data.subdata(in: start ..< start+8).value_Double); start += 8
}
values.append(value)
}
return try! Domestic.Candle(array: values, source: "")
}
}
private func build_min_db(_ productNo: String, _ candle_csvs: [URL]) {
let dataPath = URL.currentDirectory().appending(path: "data")
for csvUrl in candle_csvs {
let candleMinName = CandleMinuteFileName()
if let (_, yyyyMMdd) = candleMinName.matchedUrl(csvUrl.path), let year = Int(yyyyMMdd.prefix(4)) {
let yearDbPath = dataPath.appending(path: "\(productNo)/min/candle-\(year).db1")
//try? FileManager.default.removeItem(at: directory)
try? FileManager.default.createDirectory(at: yearDbPath, withIntermediateDirectories: true)
do {
let candles = try [Domestic.Candle].readCsv(fromFile: csvUrl)
let db = try KissDB(directory: yearDbPath)
try db.begin()
for candle in candles {
let candleData = try CandleData(candle: candle)
let item = KissDB.DataItem(key: candleData.key, value: candleData.data)
try db.insert(item: item)
if candleData.candle != candle {
assertionFailure("invalid candle data")
}
}
try db.commit()
} catch {
print("\(error)")
return
}
}
}
}
private func build_min_db_from_candle_csv() {
guard let enumerator = FileManager.subPathFiles("data") else {
return
}
var lastProductNo: String?
let candleMinName = CandleMinuteFileName()
var allCandles = [String: [URL]]()
for case let fileUrl as URL in enumerator {
guard let (productNo, yyyyMMdd) = candleMinName.matchedUrl(fileUrl.path) else {
continue
}
// Select only one product no
if lastProductNo == nil {
lastProductNo = productNo
}
else {
if lastProductNo! != productNo {
break
}
}
if allCandles.keys.contains(productNo) {
allCandles[productNo]!.append(fileUrl)
}
else {
allCandles[productNo] = [fileUrl]
}
print("product: \(productNo) \(yyyyMMdd)")
}
print("total \(allCandles.count)")
if let productCandles = allCandles.first {
build_min_db(productCandles.key, productCandles.value)
}
}
private func test_select_min_db() {
let yearDbPath = URL(filePath: "/Users/ened/Kiss/KissMe/bin/data/047040/min/candle-2023.db1")
do {
let startTime = KissDB.appTime
let db = try KissDB(directory: yearDbPath)
try db.begin()
try db.select(into: { (dataItem: KissDB.DataItem) -> Bool in
//let candleData = CandleData(key: dataItem.key, data: dataItem.value)
//print("\(candleData.candleDate) : \(candleData.candle.accumulatedTradingAmount)")
return true
})
try db.rollback()
let endTime = KissDB.appTime
print("DB count: \(db.count) insertAll elapsed: \(endTime - startTime)")
} catch {
print("\(error)")
}
}
private func test_check_name_parsed() {
let candleMinName = CandleMinuteFileName()
let url = "/Users/ened/Kiss/KissMe/bin/data/000020/min/candle-20230705.csv"
guard let (productNo, yyyyMMdd) = candleMinName.matchedUrl(url) else {
return
}
print(productNo, yyyyMMdd)
}
class CandleMinuteFileName {
let regex: NSRegularExpression
init() {
let pattern = ".*/(\\d{6})/min/candle-(\\d{8})\\.csv$"
regex = try! NSRegularExpression(pattern: pattern, options: [])
}
func matchedUrl(_ fileUrl: String) -> (productNo: String, yyyyMMdd: String)? {
let range = NSRange(location: 0, length: fileUrl.utf16.count)
let results = regex.matches(in: fileUrl, range: range)
let fragments = results.map { result in
(0 ..< result.numberOfRanges).map {
let nsRange = result.range(at: $0)
if let range = Range(nsRange, in: fileUrl) {
return String(fileUrl[range])
}
return ""
}
}
if let first = fragments.first, first.count == 3 {
return (first[1], first[2])
}
return nil
}
}

View File

@@ -17,3 +17,4 @@ import KissMe
//test_websocket_dump_data()
//test_auction()
//test_build_min_db()

View File

@@ -19,4 +19,7 @@
<FileRef
location = "container:KissMe.xcodeproj">
</FileRef>
<FileRef
location = "absolute:/Users/ened/Kiss/KissMe/libraries/KissMeme/projects/macos/KissMeme.xcodeproj">
</FileRef>
</Workspace>

View File

@@ -15,6 +15,9 @@
348168492A2F92AC00A50BD3 /* KissContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = 348168482A2F92AC00A50BD3 /* KissContext.swift */; };
349327F72A20E3E300097063 /* Foundation+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 349327F62A20E3E300097063 /* Foundation+Extensions.swift */; };
349843212A242AC900E85B08 /* KissConsole+CSV.swift in Sources */ = {isa = PBXBuildFile; fileRef = 349843202A242AC900E85B08 /* KissConsole+CSV.swift */; };
34C89F552CD6EEA90001C079 /* KissMeme.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 34C89F542CD6EEA90001C079 /* KissMeme.framework */; };
34C89F562CD6EEA90001C079 /* KissMeme.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 34C89F542CD6EEA90001C079 /* KissMeme.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
34C89F582CD6EF890001C079 /* KissConsole+DB.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34C89F572CD6EF770001C079 /* KissConsole+DB.swift */; };
34D3680D2A280801005E6756 /* KissConsole+Candle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34D3680C2A280801005E6756 /* KissConsole+Candle.swift */; };
34DA3EA42A9A176B00BB3439 /* test_websocket.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34DA3EA32A9A176B00BB3439 /* test_websocket.swift */; };
34DB3C452AA6071D00B6763E /* KissConsole+WebSocket.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34DB3C442AA6071D00B6763E /* KissConsole+WebSocket.swift */; };
@@ -40,6 +43,7 @@
dstPath = "";
dstSubfolderSpec = 10;
files = (
34C89F562CD6EEA90001C079 /* KissMeme.framework in Embed Frameworks */,
34EE76872A1C391B009761D2 /* KissMe.framework in Embed Frameworks */,
);
name = "Embed Frameworks";
@@ -59,6 +63,8 @@
349327F62A20E3E300097063 /* Foundation+Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Foundation+Extensions.swift"; sourceTree = "<group>"; };
3498431E2A24287600E85B08 /* KissMeConsoleTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = KissMeConsoleTests.swift; sourceTree = "<group>"; };
349843202A242AC900E85B08 /* KissConsole+CSV.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "KissConsole+CSV.swift"; sourceTree = "<group>"; };
34C89F542CD6EEA90001C079 /* KissMeme.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; path = KissMeme.framework; sourceTree = BUILT_PRODUCTS_DIR; };
34C89F572CD6EF770001C079 /* KissConsole+DB.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "KissConsole+DB.swift"; sourceTree = "<group>"; };
34D3680C2A280801005E6756 /* KissConsole+Candle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "KissConsole+Candle.swift"; sourceTree = "<group>"; };
34DA3EA32A9A176B00BB3439 /* test_websocket.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = test_websocket.swift; sourceTree = "<group>"; };
34DB3C442AA6071D00B6763E /* KissConsole+WebSocket.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "KissConsole+WebSocket.swift"; sourceTree = "<group>"; };
@@ -71,6 +77,7 @@
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
34C89F552CD6EEA90001C079 /* KissMeme.framework in Frameworks */,
34EE76862A1C391B009761D2 /* KissMe.framework in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
@@ -106,6 +113,7 @@
341F5F082A1463A100962D48 /* KissConsole.swift */,
34D3680C2A280801005E6756 /* KissConsole+Candle.swift */,
349843202A242AC900E85B08 /* KissConsole+CSV.swift */,
34C89F572CD6EF770001C079 /* KissConsole+DB.swift */,
3435A7F32A35B4D000D604F1 /* KissConsole+Price.swift */,
3435A7F12A35A8A900D604F1 /* KissConsole+Investor.swift */,
34EC4D1E2A7A7365002F947C /* KissConsole+News.swift */,
@@ -120,6 +128,7 @@
341F5EDA2A0A8C4600962D48 /* Frameworks */ = {
isa = PBXGroup;
children = (
34C89F542CD6EEA90001C079 /* KissMeme.framework */,
341F5EDB2A0A8C4600962D48 /* KissMe.framework */,
);
name = Frameworks;
@@ -206,6 +215,7 @@
34DA3EA42A9A176B00BB3439 /* test_websocket.swift in Sources */,
34F190132A4441F00068C697 /* KissConsole+Test.swift in Sources */,
34EC4D1F2A7A7365002F947C /* KissConsole+News.swift in Sources */,
34C89F582CD6EF890001C079 /* KissConsole+DB.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};