WIP: External asset cache

This commit is contained in:
Anuken
2026-06-01 02:41:25 -04:00
parent 8764208045
commit dc490696ac
32 changed files with 544 additions and 278 deletions
+6
View File
@@ -232,6 +232,8 @@ public class Vars implements Loadable{
public static Fi customMapDirectory;
/** data subdirectory used for custom map previews */
public static Fi mapPreviewDirectory;
/** directory for extracted assets */
public static Fi assetCacheDirectory;
/** tmp subdirectory for map conversion */
public static Fi tmpDirectory;
/** data subdirectory used for saves */
@@ -280,6 +282,7 @@ public class Vars implements Loadable{
public static GlobalVars logicVars;
public static MapEditor editor;
public static AvoidanceProcess avoidance;
public static DataAssetCache assetCache;
public static GameService service = new GameService();
public static Universe universe;
@@ -339,6 +342,7 @@ public class Vars implements Loadable{
saveDirectory = dataDirectory.child("saves/");
tmpDirectory = dataDirectory.child("tmp/");
modDirectory = dataDirectory.child("mods/");
assetCacheDirectory = dataDirectory.child("assetCache");
schematicDirectory = dataDirectory.child("schematics/");
bebuildDirectory = dataDirectory.child("be_builds/");
serverCacheFile = dataDirectory.child("server_list.json");
@@ -364,6 +368,7 @@ public class Vars implements Loadable{
fogControl = new FogControl();
bases = new BaseRegistry();
logicVars = new GlobalVars();
assetCache = new DataAssetCache();
javaPath =
new Fi(OS.prop("java.home")).child("bin/java").exists() ? new Fi(OS.prop("java.home")).child("bin/java").absolutePath() :
Core.files.local("jre/bin/java").exists() ? Core.files.local("jre/bin/java").absolutePath() : // Unix
@@ -384,6 +389,7 @@ public class Vars implements Loadable{
emptyTile = new Tile(Short.MAX_VALUE - 20, Short.MAX_VALUE - 20);
});
assetCache.load();
mods.load();
maps.load();
}
+21
View File
@@ -152,6 +152,27 @@ public class NetClient implements ApplicationListener{
finishConnecting();
});
net.handleClient(AssetRequirementStream.class, data -> {
Seq<String> required = NetworkIO.readRequiredAssets(new InflaterInputStream(data.stream));
ShortSeq missing = new ShortSeq();
for(int i = 0; i < required.size; i++){
if(!assetCache.has(required.get(i))){
missing.add((short)i);
}
}
Log.info("Requesting @ asset(s) from the server.", missing.size);
Call.requestAssets(missing.toArray());
});
net.handleClient(AssetStream.class, data -> {
Log.info("Received asset data: @ bytes.", data.stream.available());
//there's usually little reason to compress assets (PNGs, OGGs), so they are not deflated.
NetworkIO.loadAssets(data.stream);
//after receiving assets, tell the server that the client is ready to handle the world
Call.requestWorld();
});
}
public void addPacketHandler(String type, Cons<String> handler){
+45 -8
View File
@@ -21,6 +21,7 @@ import mindustry.gen.*;
import mindustry.graphics.*;
import mindustry.io.TypeIO.*;
import mindustry.logic.*;
import mindustry.mod.data.*;
import mindustry.net.*;
import mindustry.net.Administration.*;
import mindustry.net.Packets.*;
@@ -30,7 +31,6 @@ import mindustry.world.meta.*;
import java.io.*;
import java.net.*;
import java.nio.*;
import java.util.zip.*;
import static arc.util.Log.*;
import static mindustry.Vars.*;
@@ -306,7 +306,12 @@ public class NetServer implements ApplicationListener{
//playing in pvp mode automatically assigns players to teams
player.team(assignTeam(player));
sendWorldData(player);
if(state.data.hasExternalAssets()){
con.determiningAssets = true;
sendAssetRequirements(player);
}else{
sendWorldData(player);
}
platform.updateRPC();
@@ -513,13 +518,16 @@ public class NetServer implements ApplicationListener{
return assigner.assign(current, players);
}
public void sendAssetRequirements(Player player){
var stream = new ByteArrayOutputStream();
NetworkIO.writeRequiredAssets(new FastDeflaterOutputStream(stream), state.data.getAllExternalAssets());
player.con.sendStream(new AssetRequirementStream(), stream);
}
public void sendWorldData(Player player){
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DeflaterOutputStream def = new FastDeflaterOutputStream(stream);
NetworkIO.writeWorld(player, def);
WorldStream data = new WorldStream();
data.stream = new ByteArrayInputStream(stream.toByteArray());
player.con.sendStream(data);
var stream = new ByteArrayOutputStream();
NetworkIO.writeWorld(player, new FastDeflaterOutputStream(stream));
player.con.sendStream(new WorldStream(), stream);
debug("Packed @ bytes of world data to @ (@ / @)", stream.size(), player.name, player.con.address, player.uuid());
}
@@ -889,6 +897,35 @@ public class NetServer implements ApplicationListener{
}
}
@Remote(targets = Loc.client, priority = PacketPriority.high)
public static void requestWorld(Player player){
if(!player.con.hasBegunConnecting || player.con.determiningAssets || !player.con.receivingAssets || player.con.hasConnected) return;
player.con.receivingAssets = false;
netServer.sendWorldData(player);
}
@Remote(targets = Loc.client, priority = PacketPriority.high)
public static void requestAssets(Player player, short[] ids){
if(!player.con.hasBegunConnecting || !player.con.determiningAssets || player.con.receivingAssets || player.con.hasConnected) return;
player.con.determiningAssets = false;
player.con.receivingAssets = true;
Seq<DataAsset> res = new Seq<>();
Seq<DataAsset> allAssets = state.data.getAllExternalAssets();
for(short id : ids){
if(id >= allAssets.size || id < 0) continue;
res.add(allAssets.get(id));
}
var stream = new ByteArrayOutputStream();
NetworkIO.writeAssets(stream, res);
player.con.sendStream(new AssetStream(), stream);
debug("Packed @ bytes of asset data to @ (@ / @)", stream.size(), player.name, player.con.address, player.uuid());
}
@Remote(targets = Loc.client, priority = PacketPriority.high)
public static void connectConfirm(Player player){
if(player.con.kicked) return;
@@ -72,7 +72,9 @@ public class MapPatchImagesDialog extends BaseDialog{
return;
}
images.add(new ImageAsset(path, width, height, bytes));
byte[] hash = assetCache.add(bytes);
images.add(new ImageAsset(path, hash));
images.sort();
state.data.reloadImages(images);
rebuild();
@@ -93,7 +95,13 @@ public class MapPatchImagesDialog extends BaseDialog{
dialog.hide();
ui.loadAnd(() -> {
try{
Seq<Future<ImageAsset>> images = new Seq<>();
class Result{
int width, height;
Fi path;
byte[] hash; //null if width/height invalid
}
Seq<Future<?>> images = new Seq<>();
var errors = new CopyOnWriteArrayList<String>();
Fi zipped = new ZipFi(file);
@@ -101,15 +109,22 @@ public class MapPatchImagesDialog extends BaseDialog{
if(ifile.extEquals("png")){
images.add(mainExecutor.submit(() -> {
try{
Result res = new Result();
res.path = ifile;
byte[] bytes = ifile.readBytes();
Pixmap pix = new Pixmap(bytes);
int width = pix.width;
int height = pix.height;
Pixmaps.bleed(pix);
bytes = PixmapIO.writePngBytes(pix);
res.width = pix.width;
res.height = pix.height;
if(res.width < DataPatcher.maxImageSize && res.height < DataPatcher.maxImageSize){
Pixmaps.bleed(pix);
bytes = PixmapIO.writePngBytes(pix);
res.hash = assetCache.add(bytes);
}
pix.dispose();
return new ImageAsset(ifile.pathWithoutExtension(), width, height, bytes);
return res;
}catch(Throwable error){
errors.add("[accent]" + ifile.path() + "[white]: " + Strings.getSimpleMessage(error));
return null;
@@ -123,19 +138,20 @@ public class MapPatchImagesDialog extends BaseDialog{
int imported = 0;
for(var future : images){
var image = future.get();
var image = (Result)future.get();
if(image != null){
if(image.width > DataPatcher.maxImageSize || image.height > DataPatcher.maxImageSize){
errors.add("[accent]" + image.path + "[white]: " +Core.bundle.format("editor.patches.image.toolarge", width, height, DataPatcher.maxImageSize, DataPatcher.maxImageSize));
if(image.width > DataPatcher.maxImageSize || image.height > DataPatcher.maxImageSize || image.hash == null){
errors.add("[accent]" + image.path + "[white]: " + Core.bundle.format("editor.patches.image.toolarge", width, height, DataPatcher.maxImageSize, DataPatcher.maxImageSize));
continue;
}
var other = getImages().find(op -> (op.path.equalsIgnoreCase(image.path) || op.name.equalsIgnoreCase(image.name)));
String path = image.path.pathWithoutExtension(), name = image.path.nameWithoutExtension();
var other = getImages().find(op -> (op.path.equalsIgnoreCase(path) || op.name.equalsIgnoreCase(name)));
if(other != null){
errors.add("[accent]" + image.path + "[white]: " +Core.bundle.format("editor.patches.image.exists", other.name + " (" + other.path + ")").replace("\n", " "));
continue;
}
getImages().add(image);
getImages().add(new ImageAsset(path, image.hash));
imported ++;
}
}
@@ -167,8 +183,11 @@ public class MapPatchImagesDialog extends BaseDialog{
try{
try(OutputStream fos = file.write(false, 4096); ZipOutputStream zos = new ZipOutputStream(fos)){
for(var image : getImages()){
Fi cacheFile = image.getCacheFile();
if(cacheFile == null) continue;
zos.putNextEntry(new ZipEntry(image.path + ".png"));
zos.write(image.data);
zos.write(cacheFile.readBytes());
zos.closeEntry();
}
}
@@ -211,13 +230,16 @@ public class MapPatchImagesDialog extends BaseDialog{
if(searchString != null && !image.path.toLowerCase().contains(searchString)) continue;
TextureRegion region = Core.atlas.find(regionPrefix + image.name, "nomap");
boolean found = Core.atlas.has(regionPrefix + image.name);
@Nullable Fi cacheFile = image.getCacheFile();
int iwidth = found ? region.width : 1, iheight = found ? region.height : 1;
int ilength = cacheFile != null && cacheFile.exists() ? (int)cacheFile.length() : 0;
inner.table(Styles.grayPanel, t -> {
t.margin(5f);
t.top();
t.add(new BorderImage(region, 4f)).scaling(Scaling.fit).with(b -> b.drawAlpha = true).size(size - 10f).row();
t.add((found ? "" : "[red]⚠[] ") + image.name).tooltip(regionPrefix + image.name + "\n[lightgray]" + image.path).ellipsis(true).left().width(size - 10f).growX().row();
t.add(image.width + "x" + image.height).tooltip(String.format("%,d", image.data.length) + "[lightgray]b").color(Color.lightGray).left().growX().row();
t.add(iwidth + "x" + iheight).tooltip(String.format("%,d", ilength) + "[lightgray]b").color(Color.lightGray).left().growX().row();
t.table(b -> {
b.left();
b.defaults().size((size - 10f) / 4f);
@@ -259,8 +281,8 @@ public class MapPatchImagesDialog extends BaseDialog{
d.cont.add(Core.bundle.format("editor.patches.image.region.name", regionPrefix + image.name)).row();
d.cont.add(Core.bundle.format("editor.patches.image.path", image.name)).row();
d.cont.add(Core.bundle.format("editor.patches.image.env", env)).row();
d.cont.add(Core.bundle.format("editor.patches.image.size", image.width, image.height)).row();
d.cont.add(Core.bundle.format("editor.patches.image.filesize", String.format("%,d", image.data.length) + "[darkgray]b"));
d.cont.add(Core.bundle.format("editor.patches.image.size", iwidth, iheight)).row();
d.cont.add(Core.bundle.format("editor.patches.image.filesize", String.format("%,d", ilength) + "[darkgray]b"));
d.addCloseButton();
d.show();
@@ -268,7 +290,7 @@ public class MapPatchImagesDialog extends BaseDialog{
b.button(Icon.export, istyle, () -> {
platform.showFileChooser(false, "png", out -> {
try{
out.writeBytes(image.data);
image.getCacheFileNoNull().copyTo(out);
}catch(Throwable e){
ui.showException(e);
}
+6 -3
View File
@@ -48,7 +48,10 @@ public class MapIO{
public static void writeMap(Fi file, Map map) throws IOException{
try{
SaveIO.write(file, map.tags);
SaveIO.write(file, new SaveOptions(){{
extraTags = map.tags;
embedAssets = true;
}});
}catch(Exception e){
throw new IOException(e);
}
@@ -96,9 +99,9 @@ public class MapIO{
}
};
if(ver.version >= 12) ver.readRegion("patches", stream, counter, ver::skipDataPatches);
if(ver.version >= 12) ver.skipChunk(stream);
ver.readRegion("content", stream, counter, ver::readContentHeader);
if(ver.version == 11) ver.readRegion("patches", stream, counter, ver::skipDataPatches);
if(ver.version == 11) ver.skipChunk(stream);
ver.readRegion("preview_map", stream, counter, in -> ver.readMap(in, new WorldContext(){
@Override public void resize(int width, int height){}
@Override public boolean isGenerating(){return false;}
+1 -1
View File
@@ -192,7 +192,7 @@ public abstract class SaveFileReader{
public abstract void read(DataInputStream stream, CounterInputStream counter, WorldContext context) throws IOException;
public abstract void write(DataOutputStream stream) throws IOException;
public abstract void write(DataOutputStream stream, SaveOptions options) throws IOException;
public interface IORunner<T>{
void accept(T stream) throws IOException;
+5 -9
View File
@@ -111,26 +111,22 @@ public class SaveIO{
return file.sibling(file.name() + "-backup." + file.extension());
}
public static void write(Fi file, StringMap tags){
write(new FastDeflaterOutputStream(file.write(false, bufferSize)), tags);
public static void write(Fi file, SaveOptions options){
write(new FastDeflaterOutputStream(file.write(false, bufferSize)), options);
}
public static void write(Fi file){
write(file, null);
write(file, new SaveOptions());
}
public static void write(OutputStream os, StringMap tags){
public static void write(OutputStream os, SaveOptions options){
try(DataOutputStream stream = new DataOutputStream(os)){
Events.fire(new SaveWriteEvent());
SaveVersion ver = getVersion();
stream.write(header);
stream.writeInt(ver.version);
if(tags == null){
ver.write(stream);
}else{
ver.write(stream, tags);
}
ver.write(stream, options);
}catch(Throwable e){
throw new RuntimeException(e);
}
+9
View File
@@ -0,0 +1,9 @@
package mindustry.io;
import arc.struct.*;
import arc.util.*;
public class SaveOptions{
public boolean embedAssets;
public @Nullable StringMap extraTags;
}
+43 -21
View File
@@ -11,8 +11,8 @@ import mindustry.content.TechTree.*;
import mindustry.core.*;
import mindustry.ctype.*;
import mindustry.entities.*;
import mindustry.game.*;
import mindustry.game.EventType.*;
import mindustry.game.*;
import mindustry.game.Teams.*;
import mindustry.gen.*;
import mindustry.maps.Map;
@@ -59,11 +59,6 @@ public abstract class SaveVersion extends SaveFileReader{
);
}
@Override
public final void write(DataOutputStream stream) throws IOException{
write(stream, new StringMap());
}
@Override
public void read(DataInputStream stream, CounterInputStream counter, WorldContext context) throws IOException{
readRegion("meta", stream, counter, in -> readMeta(in, context));
@@ -80,9 +75,9 @@ public abstract class SaveVersion extends SaveFileReader{
}
}
public void write(DataOutputStream stream, StringMap extraTags) throws IOException{
writeRegion("meta", stream, out -> writeMeta(out, extraTags));
writeRegion("patches", stream, this::writeDataPatches);
public void write(DataOutputStream stream, SaveOptions options) throws IOException{
writeRegion("meta", stream, out -> writeMeta(out, options.extraTags));
writeRegion("patches", stream, out -> writeDataPatches(out, options.embedAssets));
writeRegion("content", stream, this::writeContentHeader);
writeRegion("map", stream, this::writeMap);
writeRegion("entities", stream, this::writeEntities);
@@ -127,7 +122,7 @@ public abstract class SaveVersion extends SaveFileReader{
}
StringMap result = new StringMap();
result.putAll(tags);
if(tags != null) result.putAll(tags);
writeStringMap(stream, result.merge(StringMap.of(
"saved", Time.millis(),
@@ -520,14 +515,6 @@ public abstract class SaveVersion extends SaveFileReader{
readWorldEntities(stream, mapping);
}
public void skipDataPatches(DataInput stream) throws IOException{
int total = stream.readInt();
for(int i = 0; i < total; i++){
DataAsset.readAsset(stream);
}
}
public void readDataPatches(DataInput stream) throws IOException{
stream.readInt(); //version - ignored for now
@@ -535,7 +522,30 @@ public abstract class SaveVersion extends SaveFileReader{
Seq<DataAsset> assets = new Seq<>(total);
for(int i = 0; i < total; i++){
assets.add(DataAsset.readAsset(stream));
byte typeId = stream.readByte();
if(typeId < 0 || typeId >= DataAssetType.all.length) throw new IOException("Invalid asset type ID: " + typeId);
String path = stream.readUTF();
boolean embedded = stream.readBoolean();
var type = DataAssetType.all[typeId];
var asset = type.create();
asset.setPath(path);
if(embedded){
asset.read(stream);
}else{
byte[] hash = new byte[32];
stream.readFully(hash);
asset.setHash(hash);
if(!asset.isCached()){
//TODO: log this when loading a save
Log.warn("Asset @: cache file not found.", asset.path);
}
}
assets.add(asset);
}
Events.fire(new DataPatchLoadEvent(assets));
@@ -543,14 +553,26 @@ public abstract class SaveVersion extends SaveFileReader{
state.data.load(assets);
}
public void writeDataPatches(DataOutput stream) throws IOException{
public void writeDataPatches(DataOutput stream, boolean forceEmbed) throws IOException{
stream.writeInt(DataPatcher.patchFormatVersion);
var assets = state.data.getAllAssets();
stream.writeInt(assets.size);
for(var asset : assets){
DataAsset.writeAsset(asset, stream);
boolean embed = forceEmbed || asset.isAlwaysEmbedded() || asset.byteHash == null;
stream.writeByte(asset.getType().ordinal());
stream.writeUTF(asset.path);
stream.writeBoolean(embed);
if(embed){
//most embedded assets (images, audio) write byte[] values directly from their cache file. game patches and content write raw string data.
asset.write(stream);
}else{
//if it's cached successfully, just write the 32-byte SHA256
stream.write(asset.byteHash);
}
}
}
+16
View File
@@ -1188,6 +1188,22 @@ public class TypeIO{
return out;
}
public static void writeShorts(Writes write, short[] ints){
write.s((short)ints.length);
for(short i : ints){
write.s(i);
}
}
public static short[] readShorts(Reads read){
short length = read.s();
short[] out = new short[length];
for(int i = 0; i < length; i++){
out[i] = read.s();
}
return out;
}
public static void writeTraceInfo(Writes write, TraceInfo trace){
writeString(write, trace.ip);
writeString(write, trace.uuid);
@@ -38,15 +38,6 @@ public class Save11 extends SaveVersion{
}
}
@Override
public void skipDataPatches(DataInput stream) throws IOException{
int amount = stream.readUnsignedByte();
for(int i = 0; i < amount; i++){
int len = stream.readInt();
stream.skipBytes(len);
}
}
//old, simplified string-only data patches
@Override
public void readDataPatches(DataInput stream) throws IOException{
+4 -20
View File
@@ -18,24 +18,6 @@ public class Save12 extends SaveVersion{
super(12);
}
@Override
public void skipDataPatches(DataInput stream) throws IOException{
stream.readInt(); //version - ignored for now
int amount = stream.readInt();
for(int i = 0; i < amount; i++){
int len = stream.readInt();
stream.skipBytes(len);
}
int imageAmount = stream.readInt();
for(int i = 0; i < imageAmount; i++){
stream.readUTF(); //name
stream.skipBytes(4); //w h
int len = stream.readInt(); //byte data
stream.skipBytes(len);
}
}
@Override
public void readDataPatches(DataInput stream) throws IOException{
stream.readInt(); //version - ignored for now
@@ -53,10 +35,12 @@ public class Save12 extends SaveVersion{
int imageAmount = stream.readInt();
for(int i = 0; i < imageAmount; i++){
String name = stream.readUTF();
short w = stream.readShort(), h = stream.readShort();
//width/height ignored
stream.readShort();
stream.readShort();
byte[] bytes = new byte[stream.readInt()];
stream.readFully(bytes);
assets.add(new ImageAsset(name, w, h, bytes));
assets.add(new ImageAsset(name, assetCache.add(bytes)));
}
Events.fire(new DataPatchLoadEvent(assets));
@@ -0,0 +1,77 @@
package mindustry.mod;
import arc.files.*;
import arc.struct.*;
import arc.util.*;
import arc.util.io.*;
import mindustry.*;
public class DataAssetCache{
private static final char[] base32Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567".toCharArray();
private ObjectMap<String, Fi> hashToFile = new ObjectMap<>();
public void load(){
//TODO: is this necessary? you could just resolve the file on the filesystem instead...
for(Fi file : Vars.assetCacheDirectory.list()){
hashToFile.put(file.name(), file);
}
}
/** @return the hash */
public byte[] add(byte[] bytes){
byte[] hash = Streams.sha256(bytes);
String name = encodeHash(hash);
Fi file = Vars.assetCacheDirectory.child(name);
file.writeBytes(bytes); //TODO: redundant disk writes if the data is already there
hashToFile.put(name, file);
return hash;
}
public boolean has(String hash){
return get(hash) != null;
}
public @Nullable Fi get(String shaHash){
if(shaHash == null) return null;
return hashToFile.get(shaHash);
}
public @Nullable Fi get(byte[] shaHash){
return get(encodeHash(shaHash));
}
//base32 without padding
public static String encodeHash(byte[] data) {
char[] out = new char[52];
int di = 0, oi = 0;
long bits;
for(int i = 0; i < 6; i++){
bits = ((long)(data[di++] & 0xFF) << 32) |
((long)(data[di++] & 0xFF) << 24) |
((long)(data[di++] & 0xFF) << 16) |
((long)(data[di++] & 0xFF) << 8) |
((long)(data[di++] & 0xFF));
out[oi++] = base32Alphabet[(int)(bits >>> 35) & 0x1F];
out[oi++] = base32Alphabet[(int)(bits >>> 30) & 0x1F];
out[oi++] = base32Alphabet[(int)(bits >>> 25) & 0x1F];
out[oi++] = base32Alphabet[(int)(bits >>> 20) & 0x1F];
out[oi++] = base32Alphabet[(int)(bits >>> 15) & 0x1F];
out[oi++] = base32Alphabet[(int)(bits >>> 10) & 0x1F];
out[oi++] = base32Alphabet[(int)(bits >>> 5) & 0x1F];
out[oi++] = base32Alphabet[(int)bits & 0x1F];
}
int b0 = data[di++] & 0xFF;
int b1 = data[di] & 0xFF;
out[oi++] = base32Alphabet[(b0 >>> 3) & 0x1F];
out[oi++] = base32Alphabet[((b0 << 2) & 0x1C) | ((b1 >>> 6) & 0x03)];
out[oi++] = base32Alphabet[(b1 >>> 1) & 0x1F];
out[oi++] = base32Alphabet[(b1 << 4) & 0x10];
return new String(out);
}
}
+5 -6
View File
@@ -6,17 +6,16 @@ import arc.util.*;
import arc.util.io.*;
import mindustry.mod.data.*;
import java.io.*;
public class DataBundleLoader{
private ObjectMap<I18NBundle, ObjectMap<String, String>> originalProperties = new ObjectMap<>();
public void load(Seq<BundleAsset> assets){
if(assets.isEmpty()) return;
ObjectMap<String, Seq<String>> localeToBundles = new ObjectMap<>();
ObjectMap<String, Seq<BundleAsset>> localeToBundles = new ObjectMap<>();
for(var asset : assets){
localeToBundles.get(asset.name, Seq::new).add(asset.string);
if(asset.getCacheFile() == null) continue;
localeToBundles.get(asset.name, Seq::new).add(asset);
}
//add new keys to each bundle
@@ -29,9 +28,9 @@ public class DataBundleLoader{
if(replacements != null){
originalProperties.put(bundle, bundle.getProperties().copy());
for(String str : replacements){
for(var asset : replacements){
try{
PropertiesUtils.load(bundle.getProperties(), new StringReader(str));
PropertiesUtils.load(bundle.getProperties(), asset.getCacheFileNoNull().reader());
}catch(Throwable e){
Log.err("Error loading bundles", e);
}
+37 -7
View File
@@ -1,6 +1,7 @@
package mindustry.mod;
import arc.*;
import arc.files.*;
import arc.graphics.*;
import arc.graphics.Texture.*;
import arc.graphics.g2d.*;
@@ -12,7 +13,9 @@ import arc.util.Log.*;
import mindustry.*;
import mindustry.mod.data.*;
import java.io.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
/** Manages data patch images. */
public class DataImagePacker{
@@ -29,15 +32,36 @@ public class DataImagePacker{
Time.mark();
float totalSumArea = 0f;
int maxSize = 0;
AtomicInteger totalSumArea = new AtomicInteger(), maxSize = new AtomicInteger();
var sizeTasks = new Seq<Future<?>>();
for(var image : images){
totalSumArea += image.width * image.height;
maxSize += Math.max(image.width, image.height);
sizeTasks.add(Vars.mainExecutor.submit(() -> {
try{
try(DataInputStream in = new DataInputStream(image.getCacheFile().read())){
long header = in.readLong();
if(header != 0x89504e470d0a1a0aL) return; //not a PNG
in.readInt(); //length
int type = in.readInt(); //chunk type
if(type != 0x49484452) return; //no HDR
int width = in.readInt();
int height = in.readInt();
if(width <= 0 || height <= 0) return; //negative size
totalSumArea.addAndGet(width * height);
synchronized(maxSize){
maxSize.set(Math.max(maxSize.get(), Math.max(width, height)));
}
}
}catch(Exception ignored){}
}));
}
int targetPower = Mathf.nextPowerOfTwo((int)(Mathf.sqrt(totalSumArea) * 1.35f));
int targetSize = Mathf.clamp(Math.max(targetPower, maxSize), 128, Math.min(4096, Vars.maxTextureSize));
Threads.awaitAll(sizeTasks);
int targetPower = Mathf.nextPowerOfTwo((int)(Mathf.sqrt(totalSumArea.get()) * 1.35f));
int targetSize = Mathf.clamp(Math.max(targetPower, maxSize.get()), 128, Math.min(4096, Vars.maxTextureSize));
PixmapPacker packer = new PixmapPacker(targetSize, targetSize, 2, true);
@@ -49,8 +73,12 @@ public class DataImagePacker{
var tasks = new Seq<Future<?>>();
for(var image : images){
tasks.add(Vars.mainExecutor.submit(() -> {
Fi cacheFile = image.getCacheFile();
//logged elsewhere
if(cacheFile == null) return;
try{
Pixmap pixmap = new Pixmap(image.data);
Pixmap pixmap = new Pixmap(cacheFile);
String name = regionPrefix + image.name;
if(anyEnv && image.path.contains("blocks/environment/")){
@@ -58,6 +86,8 @@ public class DataImagePacker{
}else{
packer.pack(name, pixmap);
}
pixmap.dispose();
}catch(Throwable e){
Log.err("Invalid patch image: " + image.path, e);
}
+41 -6
View File
@@ -5,10 +5,13 @@ import mindustry.*;
import mindustry.mod.data.*;
/**
* TODO: strict content limits and size limits.
* Images: 2mb
*
* TODO:
* - Export embedded assets in shared saves
* - Make sure steam cloud syncs the folder
* - Display missing content when loading a save
* - Clean up the class hierarchy for data assets
* - Log error when more than 65k assets are present
* - Make sure map import/export works
* - Prevent save from loading when content errors are present?
* - Patchset names are weird and broken with the new path system
* - Test planets/sectors/techtree stuff and make sure remove() works properly
@@ -25,12 +28,16 @@ public class DataManager{
private DataBundleLoader bundleLoader = new DataBundleLoader();
private ObjectMap<DataAssetType, Seq<DataAsset>> assets = new ObjectMap<>();
private Seq<DataAsset> orderedAssets = new Seq<>();
private Seq<DataAsset> orderedExternalAssets = new Seq<>();
public void reloadPatches(Seq<PatchAsset> patches){
if(patches != getPatches()) getPatches().set(patches);
patcher.unapply();
patcher.apply(patches, getContent());
rebuildOrderedAssets();
}
public void reloadImages(Seq<ImageAsset> images){
@@ -38,6 +45,8 @@ public class DataManager{
packer.unload();
packer.pack(images);
rebuildOrderedAssets();
}
public void load(Seq<DataAsset> newAssets){
@@ -57,6 +66,8 @@ public class DataManager{
}
patcher.apply(getPatches(), getContent());
rebuildOrderedAssets();
}
public void unload(){
@@ -68,12 +79,36 @@ public class DataManager{
soundLoader.unload();
assets.clear();
orderedAssets.clear();
orderedExternalAssets.clear();
}
private void rebuildOrderedAssets(){
orderedAssets.clear();
orderedExternalAssets.clear();
for(DataAssetType type : DataAssetType.all){
var seq = assets.get(type);
if(seq != null){
orderedAssets.addAll(seq);
if(!type.embedded){
orderedExternalAssets.addAll(seq);
}
}
}
}
public Seq<DataAsset> getAllAssets(){
Seq<DataAsset> result = new Seq<>();
assets.each((key, seq) -> result.addAll(seq));
return result;
return orderedAssets;
}
/** @return whether any assets like audio/images (external to saves) are loaded, requiring separate network transmission. */
public boolean hasExternalAssets(){
return orderedExternalAssets.size > 0;
}
/** @return all assets that can be external to a save (for network sync) */
public Seq<DataAsset> getAllExternalAssets(){
return orderedExternalAssets;
}
public boolean isPatched(Object content){
+19 -8
View File
@@ -2,6 +2,7 @@ package mindustry.mod;
import arc.*;
import arc.audio.*;
import arc.files.*;
import arc.struct.*;
import mindustry.*;
import mindustry.gen.*;
@@ -13,45 +14,55 @@ public class DataSoundLoader{
private Seq<Sound> loadedSounds = new Seq<>();
private Seq<Music> loadedMusic = new Seq<>();
private Seq<String> registered = new Seq<>();
public void load(Seq<SoundAsset> sounds, Seq<MusicAsset> musics){
int nextSoundId = soundIdOffset + 1;
for(var asset : sounds){
Sound sound = Vars.headless ? new Sound() : Sound.createLazy(prefix + asset.name, asset.data);
Fi file = asset.getCacheFile();
Sound sound = Vars.headless || file == null ? new Sound() : Sound.createStream(file);
loadedSounds.add(sound);
Sounds.registerSound(sound, nextSoundId ++);
if(Vars.headless || !Core.audio.initialized()) continue;
if(Vars.headless || !Core.audio.initialized() || sound.file == null) continue;
Core.assets.addAsset(sound.file.toString(), Sound.class, sound);
Core.assets.addAsset(prefix + asset.name, Sound.class, sound);
registered.add(prefix + asset.name);
}
for(var asset : musics){
Music music = Vars.headless ? new Music() : Music.createLazy(prefix + asset.name, asset.data);
Fi file = asset.getCacheFile();
Music music = Vars.headless || file == null ? new Music() : Music.create(file);
loadedMusic.add(music);
if(Vars.headless || !Core.audio.initialized()) continue;
if(Vars.headless || !Core.audio.initialized() || music.file == null) continue;
Core.assets.addAsset(music.file.toString(), Music.class, music);
Core.assets.addAsset(prefix + asset.name, Music.class, music);
registered.add(prefix + asset.name);
}
}
public void unload(){
for(var sound : loadedSounds){
if(!Vars.headless) Core.assets.unload(sound.file.toString());
sound.dispose();
Sounds.unregisterSound(sound);
}
for(var music : loadedMusic){
if(!Vars.headless) Core.assets.unload(music.file.toString());
music.dispose();
}
if(!Vars.headless){
for(String reg : registered){
Core.assets.unload(reg);
}
}
loadedSounds.clear();
loadedMusic.clear();
registered.clear();
}
}
@@ -1,32 +1,5 @@
package mindustry.mod.data;
import arc.files.*;
import java.io.*;
public abstract class AudioAsset extends DataAsset{
public byte[] data;
public abstract int maxSize();
@Override
public void readFromFile(String path, Fi file) throws IOException{
setPath(path);
if(file.length() > maxSize()) throw new IOException("Audio asset too large (" + file.length() + " bytes). Maximum length in bytes: " + maxSize());
data = file.readBytes();
}
@Override
void read(DataInput stream) throws IOException{
int len = stream.readInt();
if(len > maxSize()) throw new IOException("Audio asset too large (" + len + " bytes). Maximum length in bytes: " + maxSize());
data = new byte[len];
stream.readFully(data);
}
@Override
void write(DataOutput stream) throws IOException{
stream.writeInt(data.length);
stream.write(data);
}
}
@@ -1,36 +1,9 @@
package mindustry.mod.data;
import arc.files.*;
import arc.util.*;
import java.io.*;
public class BundleAsset extends DataAsset{
public String string = "";
@Override
public void readFromFile(String path, Fi file) throws IOException{
setPath(path);
string = file.readString();
}
@Override
public DataAssetType getType(){
return DataAssetType.bundle;
}
@Override
void read(DataInput stream) throws IOException{
int len = stream.readInt();
byte[] bytes = new byte[len];
stream.readFully(bytes);
string = new String(bytes, Strings.utf8);
}
@Override
void write(DataOutput stream) throws IOException{
byte[] bytes = string.getBytes(Strings.utf8);
stream.writeInt(bytes.length);
stream.write(bytes);
}
}
@@ -24,18 +24,13 @@ public class ContentAsset extends DataAsset{
data = file.readString();
}
@Override
public void readFromFile(String path, Fi file) throws IOException{
throw new UnsupportedOperationException("Content needs an associated type. Use the other readFromFile method.");
}
@Override
public DataAssetType getType(){
return DataAssetType.content;
}
@Override
void read(DataInput stream) throws IOException{
public void read(DataInput stream) throws IOException{
type = ContentType.all[stream.readShort()];
int len = stream.readInt();
byte[] bytes = new byte[len];
@@ -44,7 +39,7 @@ public class ContentAsset extends DataAsset{
}
@Override
void write(DataOutput stream) throws IOException{
public void write(DataOutput stream) throws IOException{
stream.writeShort(type.ordinal());
byte[] bytes = data.getBytes(Strings.utf8);
stream.writeInt(bytes.length);
+70 -24
View File
@@ -2,6 +2,8 @@ package mindustry.mod.data;
import arc.files.*;
import arc.util.*;
import mindustry.*;
import mindustry.mod.*;
import java.io.*;
@@ -12,6 +14,22 @@ public abstract class DataAsset implements Comparable<DataAsset>{
/** File name, excluding extension. This is taken from the path. */
public String name = "";
/** sha256 of the internal data. this is null for non-external assets. */
public @Nullable String stringHash;
public @Nullable byte[] byteHash;
public @Nullable Fi overrideCacheFile;
/** Caches this asset in the asset folder, and updates its hash to correspond to the appropriate cache file. */
public void updateData(byte[] data){
setHash(Vars.assetCache.add(data));
}
public void setHash(byte[] value){
if(value.length != 32) throw new IllegalArgumentException("hash must be 32 bytes long: " + value.length);
byteHash = value;
stringHash = DataAssetCache.encodeHash(value);
}
public void setPath(String path){
this.path = path.replace('\\', '/');
this.name = Strings.getFileNameWithoutExtension(path);
@@ -19,32 +37,60 @@ public abstract class DataAsset implements Comparable<DataAsset>{
public abstract DataAssetType getType();
/** Reads this asset in from a file on disk. This should perform basic validation, e.g. checking size limits, or parsing JSON. */
public abstract void readFromFile(String path, Fi file) throws IOException;
abstract void read(DataInput stream) throws IOException;
abstract void write(DataOutput stream) throws IOException;
public static DataAsset readAsset(DataInput input) throws IOException{
short typeId = input.readShort();
if(typeId < 0 || typeId >= DataAssetType.all.length) throw new IOException("Invalid asset type ID: " + typeId);
String path = input.readUTF();
var type = DataAssetType.all[typeId];
var asset = type.create();
asset.setPath(path);
asset.read(input);
return asset;
public boolean isAlwaysEmbedded(){
return getType().embedded;
}
public static void writeAsset(DataAsset asset, DataOutput output) throws IOException{
var type = asset.getType();
output.writeShort(type.ordinal());
output.writeUTF(asset.path);
asset.write(output);
public boolean isCached(){
return overrideCacheFile != null || (stringHash != null && Vars.assetCache.has(stringHash));
}
public Fi getCacheFileNoNull(){
Fi file = getCacheFile();
if(file == null) throw new RuntimeException("Cache file for asset " + path + " not found!");
return file;
}
public @Nullable Fi getCacheFile(){
return overrideCacheFile != null ? overrideCacheFile : Vars.assetCache.get(stringHash);
}
/** Reads this asset in from a file on disk. Only used on the server. */
public void readFromFile(String path, Fi file) throws IOException{
setPath(path);
setHash(file.sha256());
this.overrideCacheFile = file;
}
public void read(DataInput stream) throws IOException{
int length = stream.readInt();
if(length == 0){
Log.err("Empty asset in save: @", path);
return;
}
byte[] data = new byte[length];
stream.readFully(data);
updateData(data);
}
public void write(DataOutput stream) throws IOException{
Fi file = getCacheFile();
if(file == null || !file.exists()){
Log.err("Failed to write asset to save: missing cache file: " + path);
stream.writeInt(0);
return;
}
try{
//TODO: would be more memory efficient to use streams to copy it without reading the whole file at once
byte[] bytes = file.readBytes();
stream.writeInt(bytes.length);
stream.write(bytes);
}catch(ArcRuntimeException e){
Log.err("Failed to write asset to save: " + path, e);
stream.writeInt(0);
}
}
@Override
@@ -4,23 +4,25 @@ import arc.func.*;
import arc.struct.*;
public enum DataAssetType{
patch("patches", Seq.with("json", "hjson", "json5"), PatchAsset::new),
content("content", Seq.with("json", "hjson", "json5"), ContentAsset::new),
image("sprites", Seq.with("png"), ImageAsset::new),
sound("sounds", Seq.with("mp3", "ogg"), SoundAsset::new),
music("music", Seq.with("mp3", "ogg"), MusicAsset::new),
bundle("bundles", Seq.with("properties"), BundleAsset::new);
patch("patches", Seq.with("json", "hjson", "json5"), PatchAsset::new, true),
content("content", Seq.with("json", "hjson", "json5"), ContentAsset::new, true),
image("sprites", Seq.with("png"), ImageAsset::new, false),
sound("sounds", Seq.with("mp3", "ogg"), SoundAsset::new, false),
music("music", Seq.with("mp3", "ogg"), MusicAsset::new, false),
bundle("bundles", Seq.with("properties"), BundleAsset::new, false);
public final String folder;
public final Seq<String> extensions;
public final Prov<DataAsset> constructor;
public final boolean embedded;
public static final DataAssetType[] all = values();
DataAssetType(String folder, Seq<String> extensions, Prov<DataAsset> constructor){
DataAssetType(String folder, Seq<String> extensions, Prov<DataAsset> constructor, boolean embedded){
this.folder = folder;
this.extensions = extensions;
this.constructor = constructor;
this.embedded = embedded;
}
public DataAsset create(){
+4 -56
View File
@@ -1,64 +1,12 @@
package mindustry.mod.data;
import arc.files.*;
import mindustry.mod.*;
import java.io.*;
public class ImageAsset extends DataAsset{
/** Size of encoded image. */
public int width, height;
/** Encoded PNG data. */
public byte[] data;
public ImageAsset(String path, int width, int height, byte[] data){
public ImageAsset(){}
public ImageAsset(String path, byte[] hash){
setPath(path);
this.width = width;
this.height = height;
this.data = data;
}
ImageAsset(){}
@Override
public void readFromFile(String relativePath, Fi file) throws IOException{
byte[] data = file.readBytes();
int width, height;
//perform basic validation and fetch size from IHDR chunk
try(DataInputStream in = new DataInputStream(new ByteArrayInputStream(data))){
long header = in.readLong();
if(header != 0x89504e470d0a1a0aL){
throw new IOException("File is not a PNG.");
}
in.readInt(); //length
int type = in.readInt(); //chunk type
if(type != 0x49484452) throw new IOException("PNG files must begin with a IHDR chunk.");
width = in.readInt();
height = in.readInt();
if(width <= 0 || height <= 0) throw new IOException("PNG size must be positive.");
if(width > DataPatcher.maxImageSize || height > DataPatcher.maxImageSize) throw new IOException("PNG is larger than maximum image size (" + DataPatcher.maxImageSize + "x" + DataPatcher.maxImageSize + ")");
}
setPath(relativePath);
this.width = width;
this.height = height;
this.data = data;
}
@Override
void read(DataInput stream) throws IOException{
width = stream.readShort();
height = stream.readShort();
data = new byte[stream.readInt()];
stream.readFully(data);
}
@Override
void write(DataOutput stream) throws IOException{
stream.writeShort(width);
stream.writeShort(height);
stream.writeInt(data.length);
stream.write(data);
setHash(hash);
}
@Override
@@ -2,11 +2,6 @@ package mindustry.mod.data;
public class MusicAsset extends AudioAsset{
@Override
public int maxSize(){
return 1024 * 1024 * 10;
}
@Override
public DataAssetType getType(){
return DataAssetType.music;
+2 -2
View File
@@ -43,7 +43,7 @@ public class PatchAsset extends DataAsset{
}
@Override
void read(DataInput stream) throws IOException{
public void read(DataInput stream) throws IOException{
int len = stream.readInt();
byte[] bytes = new byte[len];
stream.readFully(bytes);
@@ -51,7 +51,7 @@ public class PatchAsset extends DataAsset{
}
@Override
void write(DataOutput stream) throws IOException{
public void write(DataOutput stream) throws IOException{
byte[] bytes = patch.getBytes(Strings.utf8);
stream.writeInt(bytes.length);
stream.write(bytes);
@@ -2,11 +2,6 @@ package mindustry.mod.data;
public class SoundAsset extends AudioAsset{
@Override
public int maxSize(){
return 1024 * 500;
}
@Override
public DataAssetType getType(){
return DataAssetType.sound;
+2
View File
@@ -46,6 +46,8 @@ public class Net{
registerPacket(StreamChunk::new);
registerPacket(WorldStream::new);
registerPacket(ConnectPacket::new);
registerPacket(AssetRequirementStream::new);
registerPacket(AssetStream::new);
//register generated packet classes
Call.registerPackets();
+7 -1
View File
@@ -33,7 +33,8 @@ public abstract class NetConnection{
/** Handles packet spam rate limits. */
public Ratekeeper packetRate = new Ratekeeper();
public boolean hasConnected, hasBegunConnecting, hasDisconnected;
//TODO: refactor to state enum
public boolean hasConnected, hasBegunConnecting, determiningAssets, receivingAssets, hasDisconnected;
public float viewWidth, viewHeight, viewX, viewY;
public NetConnection(String address){
@@ -95,6 +96,11 @@ public abstract class NetConnection{
return true;
}
public void sendStream(Streamable stream, ByteArrayOutputStream data){
stream.stream = new ByteArrayInputStream(data.toByteArray());
sendStream(stream);
}
public void sendStream(Streamable stream){
try{
int cid;
+61 -1
View File
@@ -1,6 +1,8 @@
package mindustry.net;
import arc.*;
import arc.files.*;
import arc.struct.*;
import arc.util.*;
import arc.util.io.*;
import mindustry.*;
@@ -11,6 +13,8 @@ import mindustry.gen.*;
import mindustry.io.*;
import mindustry.logic.*;
import mindustry.maps.Map;
import mindustry.mod.*;
import mindustry.mod.data.*;
import mindustry.net.Administration.*;
import mindustry.type.*;
@@ -50,7 +54,7 @@ public class NetworkIO{
stream.writeInt(player.id);
player.write(new Writes(stream));
SaveIO.getSaveWriter().writeDataPatches(stream);
SaveIO.getSaveWriter().writeDataPatches(stream, false);
SaveIO.getSaveWriter().writeContentHeader(stream);
SaveIO.getSaveWriter().writeMap(stream);
SaveIO.getSaveWriter().writeTeamBlocks(stream);
@@ -97,6 +101,62 @@ public class NetworkIO{
}
}
public static void writeRequiredAssets(OutputStream os, Seq<DataAsset> assets){
try(DataOutputStream stream = new DataOutputStream(os)){
stream.writeInt(assets.size);
for(var asset : assets){
if(asset.byteHash == null) throw new RuntimeException("Invalid asset (missing hash): " + asset.path);
stream.write(asset.byteHash);
}
}catch(IOException e){
throw new RuntimeException(e);
}
}
public static Seq<String> readRequiredAssets(InputStream is){
Seq<String> result = new Seq<>();
byte[] bytes = new byte[32];
try(DataInputStream stream = new DataInputStream(is)){
int amount = stream.readInt();
for(int i = 0; i < amount; i++){
stream.readFully(bytes);
result.add(DataAssetCache.encodeHash(bytes));
}
}catch(IOException e){
throw new RuntimeException(e);
}
return result;
}
public static void writeAssets(OutputStream os, Seq<DataAsset> assets){
try(DataOutputStream stream = new DataOutputStream(os)){
stream.writeInt(assets.size);
for(var asset : assets){
Fi file = asset.getCacheFileNoNull();
byte[] bytes = file.readBytes();
stream.writeInt(bytes.length);
stream.write(bytes);
}
}catch(IOException e){
throw new RuntimeException(e);
}
}
public static void loadAssets(InputStream is){
try(DataInputStream stream = new DataInputStream(is)){
int amount = stream.readInt();
for(int i = 0; i < amount; i++){
int len = stream.readInt();
byte[] bytes = new byte[len];
stream.readFully(bytes);
assetCache.add(bytes);
}
}catch(IOException e){
throw new RuntimeException(e);
}
}
public static ByteBuffer writeServerData(){
String name = (headless ? Config.serverName.string() : player.name);
String description = headless && !Config.desc.string().equals("off") ? Config.desc.string() : "";
+8
View File
@@ -69,6 +69,14 @@ public class Packets{
}
public static class AssetRequirementStream extends Streamable{
}
public static class AssetStream extends Streamable{
}
/** Marks the beginning of a stream. */
public static class StreamBegin extends Packet{
private static int lastid;
@@ -613,6 +613,7 @@ public class SettingsMenuDialog extends BaseDialog{
files.addAll(saveDirectory.list());
files.addAll(modDirectory.list());
files.addAll(schematicDirectory.list());
files.addAll(assetCacheDirectory.list()); //important for saves
String base = Core.settings.getDataDirectory().path();
//add directories
@@ -653,6 +654,9 @@ public class SettingsMenuDialog extends BaseDialog{
//delete old saves so they don't interfere
saveDirectory.deleteDirectory();
//clear old assets cache
assetCacheDirectory.deleteDirectory();
//purge existing tmp data, keep everything else
tmpDirectory.deleteDirectory();
+1 -1
View File
@@ -26,4 +26,4 @@ org.gradle.caching=true
org.gradle.internal.http.socketTimeout=100000
org.gradle.internal.http.connectionTimeout=100000
android.enableR8.fullMode=false
archash=a4f1529b09
archash=3888f3b183